kat0012 commited on
Commit
d2816aa
·
verified ·
1 Parent(s): 6609e2b

Parakon runtime: llama.cpp fork with fused 1-bit CUDA/Metal/CPU kernels (part 7)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +7 -0
  2. gguf-py/gguf/utility.py +340 -0
  3. gguf-py/gguf/vocab.py +933 -0
  4. gguf-py/pyproject.toml +45 -0
  5. gguf-py/tests/__init__.py +1 -0
  6. gguf-py/tests/test_gguf_reader_validation.py +37 -0
  7. gguf-py/tests/test_metadata.py +238 -0
  8. gguf-py/tests/test_quants.py +248 -0
  9. gpu_sample.csv +34 -0
  10. grammars/README.md +408 -0
  11. grammars/arithmetic.gbnf +6 -0
  12. grammars/c.gbnf +42 -0
  13. grammars/chess.gbnf +13 -0
  14. grammars/english.gbnf +6 -0
  15. grammars/japanese.gbnf +7 -0
  16. grammars/json.gbnf +25 -0
  17. grammars/json_arr.gbnf +34 -0
  18. grammars/list.gbnf +4 -0
  19. include/llama-cpp.h +30 -0
  20. include/llama.h +1638 -0
  21. licenses/LICENSE-jsonhpp +21 -0
  22. media/llama0-banner.png +3 -0
  23. media/llama0-logo.png +3 -0
  24. media/llama1-banner.png +0 -0
  25. media/llama1-icon-transparent.png +0 -0
  26. media/llama1-icon-transparent.svg +77 -0
  27. media/llama1-icon.png +0 -0
  28. media/llama1-icon.svg +87 -0
  29. media/llama1-logo.png +0 -0
  30. media/llama1-logo.svg +34 -0
  31. media/matmul.png +3 -0
  32. media/matmul.svg +1238 -0
  33. mypy.ini +7 -0
  34. out_base.txt +20 -0
  35. out_cpu.txt +0 -0
  36. out_cuda.txt +7 -0
  37. pocs/CMakeLists.txt +14 -0
  38. pocs/vdot/CMakeLists.txt +9 -0
  39. pocs/vdot/q8dot.cpp +173 -0
  40. pocs/vdot/vdot.cpp +311 -0
  41. pyproject.toml +67 -0
  42. pyrightconfig.json +22 -0
  43. requirements.txt +13 -0
  44. requirements/requirements-all.txt +18 -0
  45. requirements/requirements-compare-llama-bench.txt +3 -0
  46. requirements/requirements-convert_hf_to_gguf.txt +5 -0
  47. requirements/requirements-convert_hf_to_gguf_update.txt +1 -0
  48. requirements/requirements-convert_legacy_llama.txt +7 -0
  49. requirements/requirements-convert_llama_ggml_to_gguf.txt +1 -0
  50. requirements/requirements-convert_lora_to_gguf.txt +2 -0
.gitattributes CHANGED
@@ -35,3 +35,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  docs/android/imported-into-android-studio.jpg filter=lfs diff=lfs merge=lfs -text
37
  docs/development/llama-star/idea-arch.key filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
  docs/android/imported-into-android-studio.jpg filter=lfs diff=lfs merge=lfs -text
37
  docs/development/llama-star/idea-arch.key filter=lfs diff=lfs merge=lfs -text
38
+ media/llama0-banner.png filter=lfs diff=lfs merge=lfs -text
39
+ media/llama0-logo.png filter=lfs diff=lfs merge=lfs -text
40
+ media/matmul.png filter=lfs diff=lfs merge=lfs -text
41
+ tools/mtmd/test-1.jpeg filter=lfs diff=lfs merge=lfs -text
42
+ tools/mtmd/test-2.mp3 filter=lfs diff=lfs merge=lfs -text
43
+ tools/mtmd/test-3.mp4 filter=lfs diff=lfs merge=lfs -text
44
+ tools/mtmd/tests/test-1-positive.png filter=lfs diff=lfs merge=lfs -text
gguf-py/gguf/utility.py ADDED
@@ -0,0 +1,340 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass
4
+ from pathlib import Path
5
+ from typing import Literal
6
+
7
+ import os
8
+ import json
9
+ import numpy as np
10
+
11
+
12
+ def fill_templated_filename(filename: str, output_type: str | None) -> str:
13
+ # Given a file name fill in any type templates e.g. 'some-model-name.{ftype}.gguf'
14
+ ftype_lowercase: str = output_type.lower() if output_type is not None else ""
15
+ ftype_uppercase: str = output_type.upper() if output_type is not None else ""
16
+ return filename.format(ftype_lowercase,
17
+ outtype=ftype_lowercase, ftype=ftype_lowercase,
18
+ OUTTYPE=ftype_uppercase, FTYPE=ftype_uppercase)
19
+
20
+
21
+ def model_weight_count_rounded_notation(model_params_count: int, min_digits: int = 2) -> str:
22
+ if model_params_count > 1e12 :
23
+ # Trillions Of Parameters
24
+ scaled_model_params = model_params_count * 1e-12
25
+ scale_suffix = "T"
26
+ elif model_params_count > 1e9 :
27
+ # Billions Of Parameters
28
+ scaled_model_params = model_params_count * 1e-9
29
+ scale_suffix = "B"
30
+ elif model_params_count > 1e6 :
31
+ # Millions Of Parameters
32
+ scaled_model_params = model_params_count * 1e-6
33
+ scale_suffix = "M"
34
+ else:
35
+ # Thousands Of Parameters
36
+ scaled_model_params = model_params_count * 1e-3
37
+ scale_suffix = "K"
38
+
39
+ fix = max(min_digits - len(str(round(scaled_model_params)).lstrip('0')), 0)
40
+
41
+ return f"{scaled_model_params:.{fix}f}{scale_suffix}"
42
+
43
+
44
+ def size_label(total_params: int, shared_params: int, expert_params: int, expert_count: int) -> str:
45
+
46
+ if expert_count > 0:
47
+ pretty_size = model_weight_count_rounded_notation(abs(shared_params) + abs(expert_params), min_digits=2)
48
+ size_class = f"{expert_count}x{pretty_size}"
49
+ else:
50
+ size_class = model_weight_count_rounded_notation(abs(total_params), min_digits=2)
51
+
52
+ return size_class
53
+
54
+
55
+ def naming_convention(model_name: str | None, base_name: str | None, finetune_string: str | None, version_string: str | None, size_label: str | None, output_type: str | None, model_type: Literal['vocab', 'LoRA'] | None = None) -> str:
56
+ # Reference: https://github.com/ggml-org/ggml/blob/master/docs/gguf.md#gguf-naming-convention
57
+
58
+ if base_name is not None:
59
+ name = base_name.strip().replace(' ', '-').replace('/', '-')
60
+ elif model_name is not None:
61
+ name = model_name.strip().replace(' ', '-').replace('/', '-')
62
+ else:
63
+ name = "ggml-model"
64
+
65
+ parameters = f"-{size_label}" if size_label is not None else ""
66
+
67
+ finetune = f"-{finetune_string.strip().replace(' ', '-')}" if finetune_string is not None else ""
68
+
69
+ version = f"-{version_string.strip().replace(' ', '-')}" if version_string is not None else ""
70
+
71
+ encoding = f"-{output_type.strip().replace(' ', '-').upper()}" if output_type is not None else ""
72
+
73
+ kind = f"-{model_type.strip().replace(' ', '-')}" if model_type is not None else ""
74
+
75
+ return f"{name}{parameters}{finetune}{version}{encoding}{kind}"
76
+
77
+
78
+ @dataclass
79
+ class RemoteTensor:
80
+ dtype: str
81
+ shape: tuple[int, ...]
82
+ offset_start: int
83
+ size: int
84
+ url: str
85
+
86
+ def data(self) -> bytearray:
87
+ # TODO: handle request errors (maybe with limited retries?)
88
+ # NOTE: using a bytearray, otherwise PyTorch complains the buffer is not writeable
89
+ data = bytearray(SafetensorRemote.get_data_by_range(url=self.url, start=self.offset_start, size=self.size))
90
+ return data
91
+
92
+
93
+ class SafetensorRemote:
94
+ """
95
+ Uility class to handle remote safetensor files.
96
+ This class is designed to work with Hugging Face model repositories.
97
+
98
+ Example (one model has single safetensor file, the other has multiple):
99
+ for model_id in ["ngxson/TEST-Tiny-Llama4", "Qwen/Qwen2.5-7B-Instruct"]:
100
+ tensors = SafetensorRemote.get_list_tensors_hf_model(model_id)
101
+ print(tensors)
102
+
103
+ Example reading tensor data:
104
+ tensors = SafetensorRemote.get_list_tensors_hf_model(model_id)
105
+ for name, meta in tensors.items():
106
+ dtype, shape, offset_start, size, remote_safetensor_url = meta
107
+ # read the tensor data
108
+ data = SafetensorRemote.get_data_by_range(remote_safetensor_url, offset_start, size)
109
+ print(data)
110
+ """
111
+
112
+ BASE_DOMAIN = "https://huggingface.co"
113
+
114
+ @classmethod
115
+ def get_list_tensors_hf_model(cls, model_id: str) -> dict[str, RemoteTensor]:
116
+ """
117
+ Get list of tensors from a Hugging Face model repository.
118
+
119
+ Returns a dictionary of tensor names and their metadata.
120
+ Each tensor is represented as a tuple of (dtype, shape, offset_start, size, remote_safetensor_url)
121
+ """
122
+ # case 1: model has only one single model.safetensor file
123
+ is_single_file = cls.check_file_exist(f"{cls.BASE_DOMAIN}/{model_id}/resolve/main/model.safetensors")
124
+ if is_single_file:
125
+ url = f"{cls.BASE_DOMAIN}/{model_id}/resolve/main/model.safetensors"
126
+ return cls.get_list_tensors(url)
127
+
128
+ # case 2: model has multiple files
129
+ index_url = f"{cls.BASE_DOMAIN}/{model_id}/resolve/main/model.safetensors.index.json"
130
+ is_multiple_files = cls.check_file_exist(index_url)
131
+ if is_multiple_files:
132
+ # read the index file
133
+ index_data = cls.get_data_by_range(index_url, 0)
134
+ index_str = index_data.decode('utf-8')
135
+ index_json = json.loads(index_str)
136
+ assert index_json.get("weight_map") is not None, "weight_map not found in index file"
137
+ weight_map = index_json["weight_map"]
138
+ # get the list of files
139
+ all_files = list(set(weight_map.values()))
140
+ all_files.sort() # make sure we load shard files in order
141
+ # get the list of tensors
142
+ tensors: dict[str, RemoteTensor] = {}
143
+ for file in all_files:
144
+ url = f"{cls.BASE_DOMAIN}/{model_id}/resolve/main/{file}"
145
+ for key, val in cls.get_list_tensors(url).items():
146
+ tensors[key] = val
147
+ return tensors
148
+
149
+ raise ValueError(
150
+ f"No safetensor file has been found for model {model_id}."
151
+ "If the repo has safetensor files, make sure the model is public or you have a "
152
+ "valid Hugging Face token set in the environment variable HF_TOKEN."
153
+ )
154
+
155
+ @classmethod
156
+ def get_list_tensors(cls, url: str) -> dict[str, RemoteTensor]:
157
+ """
158
+ Get list of tensors from a remote safetensor file.
159
+
160
+ Returns a dictionary of tensor names and their metadata.
161
+ Each tensor is represented as a tuple of (dtype, shape, offset_start, size)
162
+ """
163
+ metadata, data_start_offset = cls.get_metadata(url)
164
+ res: dict[str, RemoteTensor] = {}
165
+
166
+ for name, meta in metadata.items():
167
+ if name == "__metadata__":
168
+ continue
169
+ if not isinstance(meta, dict):
170
+ raise ValueError(f"Invalid metadata for tensor '{name}': {meta}")
171
+ try:
172
+ dtype = meta["dtype"]
173
+ shape = meta["shape"]
174
+ offset_start_relative, offset_end_relative = meta["data_offsets"]
175
+ size = offset_end_relative - offset_start_relative
176
+ offset_start = data_start_offset + offset_start_relative
177
+ res[name] = RemoteTensor(dtype=dtype, shape=tuple(shape), offset_start=offset_start, size=size, url=url)
178
+ except KeyError as e:
179
+ raise ValueError(f"Missing key in metadata for tensor '{name}': {e}, meta = {meta}")
180
+
181
+ # order by name (same as default safetensors behavior)
182
+ # ref: https://github.com/huggingface/safetensors/blob/0816a1ae1d6b731cefd67f061d80d1cadd0dd7bb/bindings/python/src/lib.rs#L606
183
+ res = dict(sorted(res.items(), key=lambda t: t[0]))
184
+
185
+ return res
186
+
187
+ @classmethod
188
+ def get_metadata(cls, url: str) -> tuple[dict, int]:
189
+ """
190
+ Get JSON metadata from a remote safetensor file.
191
+
192
+ Returns tuple of (metadata, data_start_offset)
193
+ """
194
+ # Request first 5MB of the file (hopefully enough for metadata)
195
+ read_size = 5 * 1024 * 1024
196
+ raw_data = cls.get_data_by_range(url, 0, read_size)
197
+
198
+ # Parse header
199
+ # First 8 bytes contain the metadata length as u64 little-endian
200
+ if len(raw_data) < 8:
201
+ raise ValueError("Not enough data to read metadata size")
202
+ metadata_length = int.from_bytes(raw_data[:8], byteorder='little')
203
+
204
+ # Calculate the data start offset
205
+ data_start_offset = 8 + metadata_length
206
+
207
+ # Check if we have enough data to read the metadata
208
+ if len(raw_data) < 8 + metadata_length:
209
+ raise ValueError(f"Could not read complete metadata. Need {8 + metadata_length} bytes, got {len(raw_data)}")
210
+
211
+ # Extract metadata bytes and parse as JSON
212
+ metadata_bytes = raw_data[8:8 + metadata_length]
213
+ metadata_str = metadata_bytes.decode('utf-8')
214
+ try:
215
+ metadata = json.loads(metadata_str)
216
+ return metadata, data_start_offset
217
+ except json.JSONDecodeError as e:
218
+ raise ValueError(f"Failed to parse safetensor metadata as JSON: {e}")
219
+
220
+ @classmethod
221
+ def get_data_by_range(cls, url: str, start: int, size: int = -1) -> bytes:
222
+ """
223
+ Get raw byte data from a remote file by range.
224
+ If size is not specified, it will read the entire file.
225
+ """
226
+ import requests
227
+ from urllib.parse import urlparse
228
+
229
+ parsed_url = urlparse(url)
230
+ if not parsed_url.scheme or not parsed_url.netloc:
231
+ raise ValueError(f"Invalid URL: {url}")
232
+
233
+ headers = cls._get_request_headers()
234
+ if size > -1:
235
+ headers["Range"] = f"bytes={start}-{start + size}"
236
+ response = requests.get(url, allow_redirects=True, headers=headers)
237
+ response.raise_for_status()
238
+
239
+ # Get raw byte data
240
+ return response.content[slice(size if size > -1 else None)]
241
+
242
+ @classmethod
243
+ def check_file_exist(cls, url: str) -> bool:
244
+ """
245
+ Check if a file exists at the given URL.
246
+ Returns True if the file exists, False otherwise.
247
+ """
248
+ import requests
249
+ from urllib.parse import urlparse
250
+
251
+ parsed_url = urlparse(url)
252
+ if not parsed_url.scheme or not parsed_url.netloc:
253
+ raise ValueError(f"Invalid URL: {url}")
254
+
255
+ try:
256
+ headers = cls._get_request_headers()
257
+ headers["Range"] = "bytes=0-0"
258
+ response = requests.head(url, allow_redirects=True, headers=headers)
259
+ # Success (2xx) or redirect (3xx)
260
+ return 200 <= response.status_code < 400
261
+ except requests.RequestException:
262
+ return False
263
+
264
+ @classmethod
265
+ def _get_request_headers(cls) -> dict[str, str]:
266
+ """Prepare common headers for requests."""
267
+ headers = {"User-Agent": "convert_hf_to_gguf"}
268
+ if os.environ.get("HF_TOKEN"):
269
+ headers["Authorization"] = f"Bearer {os.environ['HF_TOKEN']}"
270
+ return headers
271
+
272
+
273
+ @dataclass
274
+ class LocalTensorRange:
275
+ filename: Path
276
+ offset: int
277
+ size: int
278
+
279
+
280
+ @dataclass
281
+ class LocalTensor:
282
+ dtype: str
283
+ shape: tuple[int, ...]
284
+ data_range: LocalTensorRange
285
+
286
+ def mmap_bytes(self) -> np.ndarray:
287
+ return np.memmap(self.data_range.filename, mode='c', offset=self.data_range.offset, shape=self.data_range.size)
288
+
289
+
290
+ class SafetensorsLocal:
291
+ """
292
+ Read a safetensors file from the local filesystem.
293
+
294
+ Custom parsing gives a bit more control over the memory usage.
295
+ The official safetensors library doesn't expose file ranges.
296
+ """
297
+
298
+ tensors: dict[str, LocalTensor]
299
+
300
+ def __init__(self, filename: Path):
301
+ with open(filename, "rb") as f:
302
+ metadata_length = int.from_bytes(f.read(8), byteorder='little')
303
+ file_size = os.stat(filename).st_size
304
+ if file_size < 8 + metadata_length:
305
+ raise ValueError(f"Could not read complete metadata. Need {8 + metadata_length} bytes, got {file_size}")
306
+
307
+ metadata_str = f.read(metadata_length).decode('utf-8')
308
+ try:
309
+ metadata = json.loads(metadata_str)
310
+ except json.JSONDecodeError as e:
311
+ raise ValueError(f"Failed to parse safetensors metadata as JSON: {e}")
312
+
313
+ data_start_offset = f.tell()
314
+
315
+ tensors: dict[str, LocalTensor] = {}
316
+ for name, meta in metadata.items():
317
+ if name == "__metadata__":
318
+ # ignore metadata, it's not a tensor
319
+ continue
320
+
321
+ tensors[name] = LocalTensor(
322
+ dtype=meta["dtype"],
323
+ shape=tuple(meta["shape"]),
324
+ data_range=LocalTensorRange(
325
+ filename,
326
+ data_start_offset + meta["data_offsets"][0],
327
+ meta["data_offsets"][1] - meta["data_offsets"][0],
328
+ ),
329
+ )
330
+
331
+ # order by name (same as default safetensors behavior)
332
+ # ref: https://github.com/huggingface/safetensors/blob/0816a1ae1d6b731cefd67f061d80d1cadd0dd7bb/bindings/python/src/lib.rs#L606
333
+ self.tensors = dict(sorted(tensors.items(), key=lambda t: t[0]))
334
+
335
+ def __enter__(self, *args, **kwargs):
336
+ del args, kwargs # unused
337
+ return self.tensors
338
+
339
+ def __exit__(self, *args, **kwargs):
340
+ del args, kwargs # unused
gguf-py/gguf/vocab.py ADDED
@@ -0,0 +1,933 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from enum import Enum
4
+ import re
5
+ import logging
6
+ import json
7
+ import os
8
+ from pathlib import Path
9
+ from typing import Any, Callable, Sequence, Mapping, Iterable, Protocol, ClassVar, runtime_checkable
10
+
11
+ try:
12
+ from sentencepiece import SentencePieceProcessor
13
+ except ImportError:
14
+ SentencePieceProcessor: Any = None
15
+
16
+ try:
17
+ from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # type: ignore[import-not-found, ty:unresolved-import]
18
+ from mistral_common.tokens.tokenizers.tekken import Tekkenizer # type: ignore[import-not-found, ty:unresolved-import]
19
+ from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found, ty:unresolved-import]
20
+ _filter_valid_tokenizer_files,
21
+ )
22
+ from mistral_common.tokens.tokenizers.sentencepiece import ( # type: ignore[import-not-found, ty:unresolved-import]
23
+ SentencePieceTokenizer,
24
+ )
25
+ except ImportError:
26
+ _mistral_common_installed = False
27
+ MistralTokenizer: Any = None
28
+ Tekkenizer: Any = None
29
+ SentencePieceTokenizer: Any = None
30
+ _filter_valid_tokenizer_files: Any = None
31
+ else:
32
+ _mistral_common_installed = True
33
+
34
+ try:
35
+ from mistral_common.tokens.tokenizers.utils import ( # type: ignore[import-not-found, ty:unresolved-import]
36
+ get_one_valid_tokenizer_file,
37
+ )
38
+ except ImportError:
39
+ # We still want the conversion to work with older mistral-common versions.
40
+ get_one_valid_tokenizer_file: Any = None
41
+
42
+
43
+ import gguf
44
+
45
+ from .gguf_writer import GGUFWriter
46
+
47
+ logger = logging.getLogger(__name__)
48
+
49
+
50
+ class SpecialVocab:
51
+ merges: list[str]
52
+ add_special_token: dict[str, bool]
53
+ special_token_ids: dict[str, int]
54
+ chat_template: str | Sequence[Mapping[str, str]] | None
55
+ normalizer_lowercase: bool | None
56
+ normalizer_strip_accents: bool | None
57
+
58
+ def __init__(
59
+ self, path: str | os.PathLike[str], load_merges: bool = False,
60
+ special_token_types: Iterable[str] | None = None,
61
+ n_vocab: int | None = None,
62
+ ):
63
+ self.special_token_ids = {}
64
+ self.add_special_token = {}
65
+ self.n_vocab = n_vocab
66
+ self.load_merges = load_merges
67
+ self.merges = []
68
+ self.chat_template = None
69
+ self.normalizer_lowercase = None
70
+ self.normalizer_strip_accents = None
71
+ if special_token_types is not None:
72
+ self.special_token_types = special_token_types
73
+ else:
74
+ self.special_token_types = ('bos', 'eos', 'unk', 'sep', 'pad', 'cls', 'mask')
75
+ self._load(Path(path))
76
+
77
+ def __repr__(self) -> str:
78
+ return '<SpecialVocab with {} merges, special tokens {}, add special tokens {}>'.format(
79
+ len(self.merges), self.special_token_ids or "unset", self.add_special_token or "unset",
80
+ )
81
+
82
+ def add_to_gguf(self, gw: GGUFWriter, quiet: bool = False) -> None:
83
+ if self.merges:
84
+ if not quiet:
85
+ logger.info(f'Adding {len(self.merges)} merge(s).')
86
+ gw.add_token_merges(self.merges)
87
+ elif self.load_merges:
88
+ logger.warning('Adding merges requested but no merges found, output may be non-functional.')
89
+ for typ, tokid in self.special_token_ids.items():
90
+ id_handler: Callable[[int], None] | None = getattr(gw, f'add_{typ}_token_id', None)
91
+ if id_handler is None:
92
+ logger.warning(f'No handler for special token type {typ} with id {tokid} - skipping')
93
+ continue
94
+ if not quiet:
95
+ logger.info(f'Setting special token type {typ} to {tokid}')
96
+ id_handler(tokid)
97
+ for typ, value in self.add_special_token.items():
98
+ add_handler: Callable[[bool], None] | None = getattr(gw, f'add_add_{typ}_token', None)
99
+ if add_handler is None:
100
+ logger.warning(f'No handler for add_{typ}_token with value {value} - skipping')
101
+ continue
102
+ if not quiet:
103
+ logger.info(f'Setting add_{typ}_token to {value}')
104
+ add_handler(value)
105
+ if self.chat_template is not None:
106
+ if not quiet:
107
+ logger.info(f'Setting chat_template to {self.chat_template}')
108
+ gw.add_chat_template(self.chat_template)
109
+ if self.normalizer_lowercase is not None:
110
+ if not quiet:
111
+ logger.info(f'Setting normalizer_lowercase to {self.normalizer_lowercase}')
112
+ gw.add_normalizer_lowercase(self.normalizer_lowercase)
113
+ if self.normalizer_strip_accents is not None:
114
+ if not quiet:
115
+ logger.info(f'Setting normalizer_strip_accents to {self.normalizer_strip_accents}')
116
+ gw.add_normalizer_strip_accents(self.normalizer_strip_accents)
117
+
118
+ def _load(self, path: Path) -> None:
119
+ self._try_load_from_tokenizer_json(path)
120
+ self._try_load_from_config_json(path)
121
+ if self.load_merges and not self.merges:
122
+ self._try_load_merges_txt(path)
123
+
124
+ def _try_load_merges_txt(self, path: Path) -> bool:
125
+ merges_file = path / 'merges.txt'
126
+ if not merges_file.is_file():
127
+ return False
128
+ with open(merges_file, 'r', encoding = 'utf-8') as fp:
129
+ first_line = next(fp, '').strip()
130
+ if not first_line.startswith('#'):
131
+ fp.seek(0)
132
+ line_num = 0
133
+ else:
134
+ line_num = 1
135
+ merges = []
136
+ for line in fp:
137
+ line_num += 1
138
+ line = line.strip()
139
+ if not line:
140
+ continue
141
+ parts = line.split(None, 3)
142
+ if len(parts) != 2:
143
+ logger.warning(f'{merges_file.name}: Line {line_num}: Entry malformed, ignoring')
144
+ continue
145
+ merges.append(f'{parts[0]} {parts[1]}')
146
+ self.merges = merges
147
+ return True
148
+
149
+ def _set_special_token(self, typ: str, tid: Any) -> None:
150
+ if not isinstance(tid, int):
151
+ return
152
+ if tid < 0:
153
+ raise ValueError(f'invalid value for special token type {typ}: {tid}')
154
+ if self.n_vocab is None or tid < self.n_vocab:
155
+ if typ in self.special_token_ids:
156
+ return
157
+ self.special_token_ids[typ] = tid
158
+ return
159
+ logger.warning(f'Special token type {typ}, id {tid} out of range, must be under {self.n_vocab} - skipping')
160
+
161
+ def _parse_normalizer(self, normalizer: dict) -> None:
162
+ # ref: https://huggingface.co/docs/tokenizers/api/normalizers
163
+ #
164
+ # Extracts normalizer flags from three possible formats:
165
+ # 1. Standalone: {"type": "Lowercase"}
166
+ # 2. BertNormalizer attrs: {"type": "BertNormalizer", ...}
167
+ # 3. Nested in Sequence: {"type": "Sequence", "normalizers": [...]}
168
+
169
+ normalizer_type = normalizer.get('type')
170
+ if normalizer_type == 'Lowercase':
171
+ self.normalizer_lowercase = True
172
+ elif normalizer_type == 'StripAccents':
173
+ self.normalizer_strip_accents = True
174
+ elif normalizer_type == 'BertNormalizer':
175
+ if 'lowercase' in normalizer:
176
+ self.normalizer_lowercase = normalizer['lowercase']
177
+ if 'strip_accents' in normalizer:
178
+ self.normalizer_strip_accents = normalizer['strip_accents']
179
+ elif normalizer_type == 'Sequence':
180
+ for norm in normalizer.get('normalizers', []):
181
+ self._parse_normalizer(norm)
182
+
183
+ def _try_load_from_tokenizer_json(self, path: Path) -> bool:
184
+ tokenizer = None
185
+ tokenizer_file = path / 'tokenizer.json'
186
+ if tokenizer_file.is_file():
187
+ with open(tokenizer_file, encoding = 'utf-8') as f:
188
+ tokenizer = json.load(f)
189
+ if self.load_merges:
190
+ merges = tokenizer.get('model', {}).get('merges')
191
+ if isinstance(merges, list) and merges:
192
+ if isinstance(merges[0], str):
193
+ self.merges = merges
194
+ elif isinstance(merges[0], list) and len(merges[0]) == 2 and isinstance(merges[0][0], str):
195
+ # New format since transformers 4.45 to support spaces in merges
196
+ # ref: https://github.com/ggml-org/llama.cpp/issues/9692
197
+ # TODO: internally store as the new format instead of converting to old
198
+ if any(' ' in s for pair in merges for s in pair):
199
+ logger.warning(f'Spaces in merges detected, encoding as {chr(ord(" ") + 256)!r}')
200
+ self.merges = [
201
+ ' '.join(
202
+ [
203
+ # ensure the spaces are properly encoded
204
+ ''.join(
205
+ chr(ord(c) + 256) if c == ' ' else c
206
+ for c in part
207
+ )
208
+ for part in pair
209
+ ]
210
+ )
211
+ for pair in merges
212
+ ]
213
+ else:
214
+ raise ValueError("Unknown tokenizer merges format")
215
+ # Parse normalizer configuration (e.g. Lowercase) into metadata
216
+ if normalizer := tokenizer.get('normalizer'):
217
+ self._parse_normalizer(normalizer)
218
+ added_tokens = tokenizer.get('added_tokens', {})
219
+ else:
220
+ added_tokens = {}
221
+ tokenizer_config = None
222
+ tokenizer_config_file = path / 'tokenizer_config.json'
223
+ if tokenizer_config_file.is_file():
224
+ with open(tokenizer_config_file, encoding = 'utf-8') as f:
225
+ tokenizer_config = json.load(f)
226
+ if tokenizer:
227
+ special_bos = (tokenizer_config or {}).get('bos_token')
228
+ special_cls = (tokenizer_config or {}).get('cls_token')
229
+ special_eos = (tokenizer_config or {}).get('eos_token')
230
+ special_sep = (tokenizer_config or {}).get('sep_token')
231
+ if not special_bos and special_cls and tokenizer_config:
232
+ tokenizer_config['bos_token'] = special_bos = special_cls
233
+ if not special_eos and special_sep and tokenizer_config:
234
+ tokenizer_config['eos_token'] = special_eos = special_sep
235
+ if post_processor := tokenizer.get('post_processor'):
236
+ for processor in post_processor.get('processors', [post_processor]):
237
+ if processor.get('type') == 'RobertaProcessing':
238
+ self.add_special_token['bos'] = True
239
+ self.add_special_token['eos'] = True
240
+ self.add_special_token['sep'] = True
241
+ if not special_cls and tokenizer_config:
242
+ special_cls = processor.get('cls', [special_bos])[0]
243
+ tokenizer_config['cls_token'] = special_cls
244
+ if not special_sep and tokenizer_config:
245
+ special_sep = processor.get('sep', [special_eos])[0]
246
+ tokenizer_config['sep_token'] = special_sep
247
+ continue
248
+ # Crude parsing of TemplateProcessing to determine if BOS/SEP/EOS should be added
249
+ # Only works with simple templates, **will** get it wrong on unusual sequences
250
+ if processor.get('type') == 'TemplateProcessing':
251
+ tmpl_single = processor.get('single', [])
252
+ tmpl_pair = processor.get('pair', [])
253
+ special_first = None
254
+ special_last = None
255
+ if len(tmpl_single) > 1:
256
+ if special_first := tmpl_single[0].get('SpecialToken', {}).get('id'):
257
+ if not tokenizer_config:
258
+ special_bos = special_first
259
+ elif special_first not in (special_bos, special_cls):
260
+ if not special_bos:
261
+ tokenizer_config['bos_token'] = special_bos = special_first
262
+ if not special_cls:
263
+ tokenizer_config['cls_token'] = special_cls = special_first
264
+ self.add_special_token['bos'] = True if special_first in (special_bos, special_cls) else False
265
+ if special_first not in (special_bos, special_cls):
266
+ logger.warning(f'Unknown leading special token {special_first!r} in TemplateProcessing<single>')
267
+ if special_last := tmpl_single[-1].get('SpecialToken', {}).get('id'):
268
+ if not tokenizer_config:
269
+ special_eos = special_last
270
+ elif special_last != special_eos:
271
+ if 'eot' not in self.special_token_types:
272
+ self.special_token_types = tuple(self.special_token_types) + ('eot', )
273
+ tokenizer_config['eot_token'] = special_eos
274
+ elif 'eom' not in self.special_token_types:
275
+ self.special_token_types = tuple(self.special_token_types) + ('eom', )
276
+ tokenizer_config['eom_token'] = special_eos
277
+ else:
278
+ logger.warning(f'Overriding EOS token {special_eos!r} with {special_last!r} without EOT/EOM fallback!')
279
+ tokenizer_config['eos_token'] = special_eos = special_last
280
+ self.add_special_token['eos'] = True if special_last == special_eos else False
281
+ if special_last != special_eos:
282
+ logger.warning(f'Unknown trailing special token {special_last!r} in TemplateProcessing<single>')
283
+ if tmpl_pair:
284
+ seq_start = 1 if special_first and tmpl_pair[0].get('SpecialToken', {}).get('id') == special_first else 0
285
+ seq_stop = -1 if special_last and tmpl_pair[-1].get('SpecialToken', {}).get('id') == special_last else None
286
+ if (special_first and seq_start == 0) or (special_last and seq_stop is None):
287
+ logger.warning('TemplateProcessing<single> leading/trailing special tokens do not match TemplateProcessing<pair>')
288
+ if tmpl_pair := tmpl_pair[slice(seq_start, seq_stop)]:
289
+ tmpl_a = tmpl_pair[0].get('Sequence', {}).get('id')
290
+ tmpl_b = tmpl_pair[-1].get('Sequence', {}).get('id')
291
+ if tmpl_a != 'A' or tmpl_b != 'B':
292
+ logger.warning(f'Unknown sequence {tmpl_a}...{tmpl_b} in TemplateProcessing<pair>')
293
+ # A [sep] [eos] B
294
+ if tmpl_a == 'A' and tmpl_b == 'B' and (tmpl_pair := tmpl_pair[1:-1]):
295
+ add_sep = False
296
+ if special_entry := tmpl_pair[0].get('SpecialToken', {}).get('id'):
297
+ if special_entry in (special_sep, special_eos) and not special_last:
298
+ add_sep = True
299
+ if special_entry not in (special_sep, special_eos):
300
+ logger.warning(f'Unknown separator token {special_entry!r} in TemplateProcessing<pair>')
301
+ else:
302
+ logger.warning(f'Unknown middle sequence {tmpl_pair[0]!r} in TemplateProcessing<pair>')
303
+ if len(tmpl_pair) == 2:
304
+ if special_entry := tmpl_pair[1].get('SpecialToken', {}).get('id'):
305
+ if special_entry in (special_sep, special_eos):
306
+ add_sep = True
307
+ if special_entry not in (special_sep, special_eos):
308
+ logger.warning(f'Unknown second separator token {special_entry!r} in TemplateProcessing<pair>')
309
+ else:
310
+ logger.warning(f'Unknown second middle sequence {tmpl_pair[1]!r} in TemplateProcessing<pair>')
311
+ self.add_special_token['sep'] = add_sep
312
+ if add_sep and not special_sep and tokenizer_config:
313
+ tokenizer_config['sep_token'] = special_eos
314
+ continue
315
+ if not tokenizer_config:
316
+ return True
317
+ chat_template_alt = None
318
+ chat_template_json = path / 'chat_template.json'
319
+ chat_template_jinja = path / 'chat_template.jinja'
320
+ if chat_template_jinja.is_file():
321
+ with open(chat_template_jinja, encoding = 'utf-8') as f:
322
+ chat_template_alt = f.read()
323
+ if additional_templates := list((path / 'additional_chat_templates').glob('*.jinja')):
324
+ chat_template_alt = [{'name': 'default', 'template': chat_template_alt}]
325
+ for template_path in additional_templates:
326
+ with open(template_path, encoding = 'utf-8') as fp:
327
+ chat_template_alt.append({'name': template_path.stem, 'template': fp.read()})
328
+ elif chat_template_json.is_file():
329
+ with open(chat_template_json, encoding = 'utf-8') as f:
330
+ chat_template_alt = json.load(f).get('chat_template')
331
+ chat_template = tokenizer_config.get('chat_template', chat_template_alt)
332
+ if chat_template is None or isinstance(chat_template, (str, list)):
333
+ self.chat_template = chat_template
334
+ else:
335
+ logger.warning(f'Bad type for chat_template field in {tokenizer_config_file!r} - ignoring')
336
+ for typ in self.special_token_types:
337
+ add_entry = tokenizer_config.get(f'add_{typ}_token')
338
+ if isinstance(add_entry, bool):
339
+ self.add_special_token[typ] = add_entry
340
+ entry = tokenizer_config.get(f'{typ}_token')
341
+ if isinstance(entry, str):
342
+ tc_content = entry
343
+ elif isinstance(entry, dict):
344
+ entry_content = entry.get('content')
345
+ if not isinstance(entry_content, str):
346
+ continue
347
+ tc_content = entry_content
348
+ else:
349
+ continue
350
+ # We only need the first match here.
351
+ maybe_token_id = next(
352
+ (atok.get('id') for atok in added_tokens if atok.get('content') == tc_content),
353
+ None,
354
+ )
355
+ self._set_special_token(typ, maybe_token_id)
356
+ return True
357
+
358
+ def _try_load_from_config_json(self, path: Path) -> bool:
359
+ config_file = path / 'config.json'
360
+ if not config_file.is_file():
361
+ return False
362
+ with open(config_file, encoding = 'utf-8') as f:
363
+ config = json.load(f)
364
+ for typ in self.special_token_types:
365
+ token_id = config.get(f'{typ}_token_id')
366
+ # If not found at root, check in text_config (for multimodal models like Kimi-VL)
367
+ if token_id is None and 'text_config' in config:
368
+ token_id = config['text_config'].get(f'{typ}_token_id')
369
+ self._set_special_token(typ, token_id)
370
+ return True
371
+
372
+
373
+ @runtime_checkable
374
+ class BaseVocab(Protocol):
375
+ tokenizer_model: ClassVar[str]
376
+ name: ClassVar[str]
377
+
378
+
379
+ @runtime_checkable
380
+ class Vocab(BaseVocab, Protocol):
381
+ vocab_size: int
382
+ added_tokens_dict: dict[str, int]
383
+ added_tokens_list: list[str]
384
+ fname_tokenizer: Path
385
+
386
+ def __init__(self, base_path: Path): ...
387
+ def all_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]: ...
388
+
389
+
390
+ class NoVocab(BaseVocab):
391
+ tokenizer_model = "no_vocab"
392
+ name = "no_vocab"
393
+
394
+ def __repr__(self) -> str:
395
+ return "<NoVocab for a model without integrated vocabulary>"
396
+
397
+
398
+ class BpeVocab(Vocab):
399
+ tokenizer_model = "gpt2"
400
+ name = "bpe"
401
+
402
+ def __init__(self, base_path: Path):
403
+ added_tokens: dict[str, int] = {}
404
+
405
+ if (fname_tokenizer := base_path / 'vocab.json').exists():
406
+ # "slow" tokenizer
407
+ with open(fname_tokenizer, encoding="utf-8") as f:
408
+ self.vocab = json.load(f)
409
+
410
+ try:
411
+ # FIXME: Verify that added tokens here _cannot_ overlap with the main vocab.
412
+ with open(base_path / 'added_tokens.json', encoding="utf-8") as f:
413
+ added_tokens = json.load(f)
414
+ except FileNotFoundError:
415
+ pass
416
+ else:
417
+ # "fast" tokenizer
418
+ fname_tokenizer = base_path / 'tokenizer.json'
419
+
420
+ # if this fails, FileNotFoundError propagates to caller
421
+ with open(fname_tokenizer, encoding="utf-8") as f:
422
+ tokenizer_json = json.load(f)
423
+
424
+ tokenizer_model: dict[str, Any] = tokenizer_json['model']
425
+ if (
426
+ tokenizer_model['type'] != 'BPE' or tokenizer_model.get('byte_fallback', False)
427
+ or tokenizer_json['decoder']['type'] != 'ByteLevel'
428
+ ):
429
+ raise FileNotFoundError('Cannot find GPT-2 BPE tokenizer')
430
+
431
+ self.vocab = tokenizer_model["vocab"]
432
+
433
+ if (added := tokenizer_json.get('added_tokens')) is not None:
434
+ # Added tokens here can be duplicates of the main vocabulary.
435
+ added_tokens = {item['content']: item['id']
436
+ for item in added
437
+ if item['content'] not in self.vocab}
438
+
439
+ vocab_size = len(self.vocab)
440
+ expected_ids = list(range(vocab_size, vocab_size + len(added_tokens)))
441
+ actual_ids = sorted(added_tokens.values())
442
+ if expected_ids != actual_ids:
443
+ expected_end_id = vocab_size + len(actual_ids) - 1
444
+ raise ValueError(f"Expected the {len(actual_ids)} added token ID(s) to be sequential in the range "
445
+ f"{vocab_size} - {expected_end_id}; got {actual_ids}")
446
+
447
+ items = sorted(added_tokens.items(), key=lambda text_idx: text_idx[1])
448
+ self.added_tokens_dict = added_tokens
449
+ self.added_tokens_list = [text for (text, idx) in items]
450
+ self.vocab_size_base = vocab_size
451
+ self.vocab_size = self.vocab_size_base + len(self.added_tokens_list)
452
+ self.fname_tokenizer = fname_tokenizer
453
+
454
+ def bpe_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
455
+ reverse_vocab = {id: encoded_tok for encoded_tok, id in self.vocab.items()}
456
+
457
+ for i, _ in enumerate(self.vocab):
458
+ yield reverse_vocab[i], 0.0, gguf.TokenType.NORMAL
459
+
460
+ def added_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
461
+ for text in self.added_tokens_list:
462
+ score = -1000.0
463
+ yield text.encode("utf-8"), score, gguf.TokenType.CONTROL
464
+
465
+ def all_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
466
+ yield from self.bpe_tokens()
467
+ yield from self.added_tokens()
468
+
469
+ def __repr__(self) -> str:
470
+ return f"<BpeVocab with {self.vocab_size_base} base tokens and {len(self.added_tokens_list)} added tokens>"
471
+
472
+
473
+ class SentencePieceVocab(Vocab):
474
+ tokenizer_model = "llama"
475
+ name = "spm"
476
+
477
+ def __init__(self, base_path: Path):
478
+ if SentencePieceProcessor is None:
479
+ raise RuntimeError("sentencepiece is not installed")
480
+
481
+ added_tokens: dict[str, int] = {}
482
+ if (fname_tokenizer := base_path / 'tokenizer.model').exists():
483
+ # normal location
484
+ try:
485
+ with open(base_path / 'added_tokens.json', encoding="utf-8") as f:
486
+ added_tokens = json.load(f)
487
+ except FileNotFoundError:
488
+ pass
489
+ elif not (fname_tokenizer := base_path.parent / 'tokenizer.model').exists():
490
+ # not found in alternate location either
491
+ raise FileNotFoundError('Cannot find tokenizer.model')
492
+
493
+ self.sentencepiece_tokenizer = SentencePieceProcessor()
494
+ self.sentencepiece_tokenizer.LoadFromFile(str(fname_tokenizer))
495
+ vocab_size = self.sentencepiece_tokenizer.vocab_size()
496
+
497
+ new_tokens = {id: piece for piece, id in added_tokens.items() if id >= vocab_size}
498
+ expected_new_ids = list(range(vocab_size, vocab_size + len(new_tokens)))
499
+ actual_new_ids = sorted(new_tokens.keys())
500
+
501
+ if expected_new_ids != actual_new_ids:
502
+ raise ValueError(f"Expected new token IDs {expected_new_ids} to be sequential; got {actual_new_ids}")
503
+
504
+ # Token pieces that were added to the base vocabulary.
505
+ self.added_tokens_dict = added_tokens
506
+ self.added_tokens_list = [new_tokens[id] for id in actual_new_ids]
507
+ self.vocab_size_base = vocab_size
508
+ self.vocab_size = self.vocab_size_base + len(self.added_tokens_list)
509
+ self.fname_tokenizer = fname_tokenizer
510
+
511
+ def sentencepiece_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
512
+ tokenizer = self.sentencepiece_tokenizer
513
+ for i in range(tokenizer.vocab_size()):
514
+ piece = tokenizer.IdToPiece(i)
515
+ text = piece.encode("utf-8")
516
+ score: float = tokenizer.GetScore(i)
517
+
518
+ toktype = gguf.TokenType.NORMAL
519
+ if tokenizer.IsUnknown(i):
520
+ toktype = gguf.TokenType.UNKNOWN
521
+ if tokenizer.IsControl(i):
522
+ toktype = gguf.TokenType.CONTROL
523
+
524
+ # NOTE: I think added_tokens are user defined.
525
+ # ref: https://github.com/google/sentencepiece/blob/master/src/sentencepiece_model.proto
526
+ # if tokenizer.is_user_defined(i): toktype = gguf.TokenType.USER_DEFINED
527
+
528
+ if tokenizer.IsUnused(i):
529
+ toktype = gguf.TokenType.UNUSED
530
+ if tokenizer.IsByte(i):
531
+ toktype = gguf.TokenType.BYTE
532
+
533
+ yield text, score, toktype
534
+
535
+ def added_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
536
+ for text in self.added_tokens_list:
537
+ score = -1000.0
538
+ yield text.encode("utf-8"), score, gguf.TokenType.USER_DEFINED
539
+
540
+ def all_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
541
+ yield from self.sentencepiece_tokens()
542
+ yield from self.added_tokens()
543
+
544
+ def __repr__(self) -> str:
545
+ return f"<SentencePieceVocab with {self.vocab_size_base} base tokens and {len(self.added_tokens_list)} added tokens>"
546
+
547
+
548
+ class LlamaHfVocab(Vocab):
549
+ tokenizer_model = "llama"
550
+ name = "hfft"
551
+
552
+ def __init__(self, base_path: Path):
553
+ fname_tokenizer = base_path / 'tokenizer.json'
554
+ # if this fails, FileNotFoundError propagates to caller
555
+ with open(fname_tokenizer, encoding='utf-8') as f:
556
+ tokenizer_json = json.load(f)
557
+
558
+ # pre-check so we know if we need transformers
559
+ tokenizer_model: dict[str, Any] = tokenizer_json['model']
560
+ is_llama3 = (
561
+ tokenizer_model['type'] == 'BPE' and tokenizer_model.get('ignore_merges', False)
562
+ and not tokenizer_model.get('byte_fallback', True)
563
+ )
564
+ if is_llama3:
565
+ raise TypeError('Llama 3 must be converted with BpeVocab')
566
+
567
+ if not is_llama3 and (
568
+ tokenizer_model['type'] != 'BPE' or not tokenizer_model.get('byte_fallback', False)
569
+ or tokenizer_json['decoder']['type'] != 'Sequence'
570
+ ):
571
+ raise FileNotFoundError('Cannot find Llama BPE tokenizer')
572
+
573
+ try:
574
+ from transformers import AutoTokenizer
575
+ except ImportError as e:
576
+ raise ImportError(
577
+ "To use LlamaHfVocab, please install the `transformers` package. "
578
+ "You can install it with `pip install transformers`."
579
+ ) from e
580
+
581
+ # Allow the tokenizer to default to slow or fast versions.
582
+ # Explicitly set tokenizer to use local paths.
583
+ self.tokenizer = AutoTokenizer.from_pretrained(
584
+ base_path,
585
+ cache_dir=base_path,
586
+ local_files_only=True,
587
+ )
588
+ assert self.tokenizer.is_fast # assume tokenizer.json is used # ty: ignore[unresolved-attribute]
589
+
590
+ # Initialize lists and dictionaries for added tokens
591
+ self.added_tokens_list = []
592
+ self.added_tokens_dict = dict()
593
+ self.added_tokens_ids = set()
594
+
595
+ # Process added tokens
596
+ for tok, tokidx in sorted(
597
+ self.tokenizer.get_added_vocab().items(), key=lambda x: x[1] # ty: ignore[unresolved-attribute]
598
+ ):
599
+ # Only consider added tokens that are not in the base vocabulary
600
+ if tokidx >= self.tokenizer.vocab_size: # ty: ignore[unresolved-attribute]
601
+ self.added_tokens_list.append(tok)
602
+ self.added_tokens_dict[tok] = tokidx
603
+ self.added_tokens_ids.add(tokidx)
604
+
605
+ # Store special tokens and their IDs
606
+ self.specials = {
607
+ tok: self.tokenizer.get_vocab()[tok] # ty: ignore[unresolved-attribute]
608
+ for tok in self.tokenizer.all_special_tokens # ty: ignore[unresolved-attribute]
609
+ }
610
+ self.special_ids = set(self.tokenizer.all_special_ids) # ty: ignore[unresolved-attribute]
611
+
612
+ # Set vocabulary sizes
613
+ self.vocab_size_base = self.tokenizer.vocab_size # ty: ignore[unresolved-attribute]
614
+ self.vocab_size = self.vocab_size_base + len(self.added_tokens_list)
615
+
616
+ self.fname_tokenizer = fname_tokenizer
617
+
618
+ def hf_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
619
+ reverse_vocab = {
620
+ id: encoded_tok for encoded_tok, id in self.tokenizer.get_vocab().items() # ty: ignore[unresolved-attribute]
621
+ }
622
+
623
+ for token_id in range(self.vocab_size_base):
624
+ # Skip processing added tokens here
625
+ if token_id in self.added_tokens_ids:
626
+ continue
627
+
628
+ # Convert token text to bytes
629
+ token_text = reverse_vocab[token_id].encode("utf-8")
630
+
631
+ # Yield token text, score, and type
632
+ yield token_text, self.get_token_score(token_id), self.get_token_type(
633
+ token_id, token_text, self.special_ids # Reuse already stored special IDs
634
+ )
635
+
636
+ def get_token_type(self, token_id: int, token_text: bytes, special_ids: set[int]) -> gguf.TokenType:
637
+ # Special case for byte tokens
638
+ if re.fullmatch(br"<0x[0-9A-Fa-f]{2}>", token_text):
639
+ return gguf.TokenType.BYTE
640
+
641
+ # Determine token type based on whether it's a special token
642
+ return gguf.TokenType.CONTROL if token_id in special_ids else gguf.TokenType.NORMAL
643
+
644
+ def get_token_score(self, token_id: int) -> float:
645
+ # Placeholder for actual logic to determine the token's score
646
+ # This needs to be implemented based on specific requirements
647
+ return -1000.0 # Default score
648
+
649
+ def added_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
650
+ for text in self.added_tokens_list:
651
+ if text in self.specials:
652
+ toktype = self.get_token_type(self.specials[text], b'', self.special_ids)
653
+ score = self.get_token_score(self.specials[text])
654
+ else:
655
+ toktype = gguf.TokenType.USER_DEFINED
656
+ score = -1000.0
657
+
658
+ yield text.encode("utf-8"), score, toktype
659
+
660
+ def has_newline_token(self):
661
+ return "<0x0A>" in self.tokenizer.vocab or "\n" in self.tokenizer.vocab # ty: ignore[unresolved-attribute]
662
+
663
+ def all_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
664
+ yield from self.hf_tokens()
665
+ yield from self.added_tokens()
666
+
667
+ def __repr__(self) -> str:
668
+ return f"<LlamaHfVocab with {self.vocab_size_base} base tokens and {len(self.added_tokens_list)} added tokens>"
669
+
670
+
671
+ class MistralTokenizerType(str, Enum):
672
+ spm = "spm"
673
+ tekken = "tekken"
674
+
675
+
676
+ # Copied from Transformers (Apache 2.0)
677
+ # https://github.com/huggingface/transformers/blob/main/src/transformers/convert_slow_tokenizer.py#L1544
678
+
679
+ def bytes_to_unicode() -> dict[int, str]:
680
+ """
681
+ Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
682
+ characters the bpe code barfs on.
683
+
684
+ The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
685
+ if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
686
+ decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
687
+ tables between utf-8 bytes and unicode strings.
688
+ """
689
+ bs = (
690
+ list(range(ord("!"), ord("~") + 1))
691
+ + list(range(ord("¡"), ord("¬") + 1))
692
+ + list(range(ord("®"), ord("ÿ") + 1))
693
+ )
694
+ cs = bs[:]
695
+ n = 0
696
+ for b in range(2**8):
697
+ if b not in bs:
698
+ bs.append(b)
699
+ cs.append(2**8 + n)
700
+ n += 1
701
+ cs_str = [chr(n) for n in cs]
702
+ return dict(zip(bs, cs_str))
703
+
704
+
705
+ class MistralVocab(Vocab):
706
+ tokenizer_model = "mistral"
707
+ name = "mistral"
708
+
709
+ added_tokens_dict: dict[str, int] = {}
710
+ added_tokens_list: list[str] = []
711
+
712
+ def __init__(self, base_path: Path):
713
+ if not _mistral_common_installed:
714
+ raise ImportError(
715
+ "To use MistralVocab, please install the `mistral-common` package. "
716
+ "You can install it with `pip install mistral-common`."
717
+ )
718
+ assert _filter_valid_tokenizer_files is not None, "mistral_common is not installed"
719
+ assert MistralTokenizer is not None, "mistral_common is not installed"
720
+ assert Tekkenizer is not None, "mistral_common is not installed"
721
+
722
+ logger.info(f"Loading Mistral tokenizer from {base_path}")
723
+
724
+ # Find the tokenizer files
725
+ all_files = [f.as_posix() for f in base_path.glob("**/*") if f.is_file()]
726
+
727
+ if get_one_valid_tokenizer_file is not None:
728
+ tokenizer_file_path = get_one_valid_tokenizer_file(all_files)
729
+ else:
730
+ valid_tokenizer_files = _filter_valid_tokenizer_files(all_files)
731
+
732
+ if len(valid_tokenizer_files) == 0:
733
+ raise ValueError(f"No tokenizer file found in the directory: {base_path}")
734
+ # If there are multiple tokenizer files, we use tekken.json if it exists, otherwise the versioned one.
735
+ if len(valid_tokenizer_files) > 1:
736
+ if "tekken.json" in valid_tokenizer_files:
737
+ tokenizer_file = "tekken.json"
738
+ else:
739
+ tokenizer_file = sorted(valid_tokenizer_files)[-1]
740
+ logger.warning(
741
+ f"Multiple tokenizer files found in {base_path}. Using {tokenizer_file}"
742
+ )
743
+ else:
744
+ tokenizer_file = valid_tokenizer_files[0]
745
+
746
+ tokenizer_file_path = base_path / tokenizer_file
747
+
748
+ self.tokenizer: Any = MistralTokenizer.from_file(
749
+ tokenizer_file_path
750
+ ).instruct_tokenizer.tokenizer
751
+ self.tokenizer_type = (
752
+ MistralTokenizerType.tekken
753
+ if isinstance(self.tokenizer, Tekkenizer)
754
+ else MistralTokenizerType.spm
755
+ )
756
+ self.vocab_size = self.tokenizer.n_words
757
+ self.fname_tokenizer = tokenizer_file_path
758
+ self._name = (
759
+ "mistral-" + self.tokenizer_type.value + "-" + self.tokenizer.version
760
+ )
761
+
762
+ @property
763
+ def tokenizer_name(self) -> str:
764
+ return self._name
765
+
766
+ @property
767
+ def gguf_tokenizer_model(self) -> str:
768
+ return "llama" if self.tokenizer_type == MistralTokenizerType.spm else "gpt2"
769
+
770
+ def _sentencepiece_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
771
+ assert SentencePieceTokenizer is not None, "mistral_common is not installed"
772
+ assert isinstance(self.tokenizer, SentencePieceTokenizer), (
773
+ f"Expected SentencePieceTokenizer, got {type(self.tokenizer)}"
774
+ )
775
+
776
+ for i in range(self.tokenizer._model.vocab_size()):
777
+ piece = self.tokenizer._model.IdToPiece(i)
778
+ text = piece.encode("utf-8")
779
+ score: float = self.tokenizer._model.GetScore(i)
780
+
781
+ toktype = gguf.TokenType.NORMAL
782
+ if self.tokenizer._model.IsUnknown(i):
783
+ toktype = gguf.TokenType.UNKNOWN
784
+ if self.tokenizer._model.IsControl(i):
785
+ toktype = gguf.TokenType.CONTROL
786
+
787
+ if self.tokenizer._model.IsUnused(i):
788
+ toktype = gguf.TokenType.UNUSED
789
+ if self.tokenizer._model.IsByte(i):
790
+ toktype = gguf.TokenType.BYTE
791
+
792
+ yield text, score, toktype
793
+
794
+ def _tekken_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
795
+ assert Tekkenizer is not None, "mistral_common is not installed"
796
+ assert isinstance(self.tokenizer, Tekkenizer), (
797
+ f"Expected Tekkenizer, got {type(self.tokenizer)}"
798
+ )
799
+
800
+ byte_encoder = bytes_to_unicode()
801
+ for token_id in range(self.tokenizer.num_special_tokens):
802
+ yield (
803
+ self.tokenizer.id_to_piece(token_id).encode("utf-8"),
804
+ 0,
805
+ gguf.TokenType.CONTROL
806
+ )
807
+ for token in self.tokenizer._tekken_token2id_nospecial:
808
+ yield (
809
+ self.token_bytes_to_string(token, byte_encoder).encode("utf-8"),
810
+ 0,
811
+ gguf.TokenType.NORMAL,
812
+ )
813
+
814
+ def get_token_id(self, token: str) -> int:
815
+ assert SentencePieceTokenizer is not None and Tekkenizer is not None, "mistral_common is not installed"
816
+ if self.tokenizer_type == MistralTokenizerType.spm:
817
+ assert isinstance(self.tokenizer, SentencePieceTokenizer)
818
+ return self.tokenizer._vocab.index(token)
819
+ elif self.tokenizer_type == MistralTokenizerType.tekken:
820
+ assert isinstance(self.tokenizer, Tekkenizer)
821
+ return (
822
+ self.tokenizer._vocab.index(token) + self.tokenizer.num_special_tokens
823
+ )
824
+ else:
825
+ raise ValueError(f"Unknown tokenizer type: {self.tokenizer_type}")
826
+
827
+ @property
828
+ def bos_id(self) -> int:
829
+ return self.tokenizer.bos_id
830
+
831
+ @property
832
+ def eos_id(self) -> int:
833
+ return self.tokenizer.eos_id
834
+
835
+ @property
836
+ def pad_id(self) -> int:
837
+ if self.tokenizer.pad_id == -1:
838
+ return self.eos_id
839
+ return self.tokenizer.pad_id
840
+
841
+ @property
842
+ def unk_id(self) -> int:
843
+ return self.tokenizer.unk_id
844
+
845
+ @property
846
+ def bos_token(self) -> str:
847
+ return self.tokenizer.id_to_piece(self.tokenizer.bos_id)
848
+
849
+ @property
850
+ def eos_token(self) -> str:
851
+ return self.tokenizer.id_to_piece(self.tokenizer.eos_id)
852
+
853
+ @property
854
+ def pad_token(self) -> str:
855
+ return self.tokenizer.id_to_piece(self.tokenizer.pad_id)
856
+
857
+ @property
858
+ def unk_token(self) -> str:
859
+ return self.tokenizer.id_to_piece(self.tokenizer.unk_id)
860
+
861
+ def all_tokens(self) -> Iterable[tuple[bytes, float, gguf.TokenType]]:
862
+ if self.tokenizer_type == MistralTokenizerType.spm:
863
+ yield from self._sentencepiece_tokens()
864
+
865
+ elif self.tokenizer_type == MistralTokenizerType.tekken:
866
+ yield from self._tekken_tokens()
867
+
868
+ else:
869
+ raise ValueError(f"Unknown tokenizer type: {self.tokenizer_type}")
870
+
871
+ @staticmethod
872
+ def token_bytes_to_string(b, byte_encoder):
873
+ return "".join([byte_encoder[ord(char)] for char in b.decode("latin-1")])
874
+
875
+ def extract_vocab_merges_from_model(self):
876
+ # Adapted from Transformers (Apache 2.0)
877
+ # https://github.com/huggingface/transformers/blob/main/src/transformers/convert_slow_tokenizer.py
878
+ assert Tekkenizer is not None and isinstance(self.tokenizer, Tekkenizer), (
879
+ f"Expected Tekkenizer, got {type(self.tokenizer)}"
880
+ )
881
+ mergeable_ranks = self.tokenizer._model._mergeable_ranks
882
+ token_bytes_map = {
883
+ rank: token_bytes for token_bytes, rank in mergeable_ranks.items()
884
+ }
885
+ merge_pairs = []
886
+
887
+ # Sort vocab by rank to ensure correct merge order
888
+ for i in range(256, self.vocab_size - self.tokenizer.num_special_tokens):
889
+ merged_token = token_bytes_map[i]
890
+ local = []
891
+ for j in range(1, len(merged_token)):
892
+ left = merged_token[:j]
893
+ right = merged_token[j:]
894
+ if (
895
+ left in mergeable_ranks
896
+ and right in mergeable_ranks
897
+ and (left + right) in mergeable_ranks
898
+ ):
899
+ local.append((left, right, i))
900
+ if not local:
901
+ raise ValueError(
902
+ f"Could not find valid merge for token at rank {i}: {merged_token.decode('latin-1')}"
903
+ )
904
+ local = sorted(
905
+ local,
906
+ key=lambda x: (mergeable_ranks[x[0]], mergeable_ranks[x[1]]),
907
+ reverse=False,
908
+ )
909
+ merge_pairs.extend(local)
910
+ merge_pairs = sorted(merge_pairs, key=lambda val: val[2], reverse=False)
911
+
912
+ byte_encoder = bytes_to_unicode()
913
+
914
+ decoded_merge_pairs = [
915
+ [
916
+ self.token_bytes_to_string(val[0], byte_encoder),
917
+ self.token_bytes_to_string(val[1], byte_encoder),
918
+ ]
919
+ for val in merge_pairs
920
+ ]
921
+
922
+ merges = [
923
+ " ".join(
924
+ [
925
+ # ensure the spaces are properly encoded
926
+ "".join(chr(ord(c) + 256) if c == " " else c for c in part)
927
+ for part in pair
928
+ ]
929
+ )
930
+ for pair in decoded_merge_pairs
931
+ ]
932
+
933
+ return merges
gguf-py/pyproject.toml ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [project]
2
+ name = "gguf"
3
+ version = "0.19.0"
4
+ description = "Read and write ML models in GGUF for GGML"
5
+ keywords = ["ggml", "gguf", "llama.cpp"]
6
+ dynamic = ["classifiers"]
7
+ readme = "README.md"
8
+ authors = [{name = "GGML", email = "ggml@ggml.ai"}]
9
+ requires-python = '>=3.10'
10
+ dependencies = ['numpy (>=1.17)', 'tqdm (>=4.27)', 'pyyaml (>=5.1)', 'requests (>=2.25)']
11
+ classifiers = [
12
+ "Programming Language :: Python :: 3",
13
+ "License :: OSI Approved :: MIT License",
14
+ "Operating System :: OS Independent",
15
+ ]
16
+
17
+ [project.urls]
18
+ homepage = "https://ggml.ai"
19
+ repository = "https://github.com/ggml-org/llama.cpp"
20
+
21
+ [project.scripts]
22
+ gguf-convert-endian = "gguf.scripts.gguf_convert_endian:main"
23
+ gguf-dump = "gguf.scripts.gguf_dump:main"
24
+ gguf-set-metadata = "gguf.scripts.gguf_set_metadata:main"
25
+ gguf-new-metadata = "gguf.scripts.gguf_new_metadata:main"
26
+ gguf-editor-gui = "gguf.scripts.gguf_editor_gui:main"
27
+
28
+ [project.optional-dependencies]
29
+ gui = ['PySide6 (>=6.9,<7.0) ; python_version >= "3.9" and python_version < "3.14"']
30
+
31
+ [tool.poetry]
32
+ packages = [
33
+ {include = "gguf"},
34
+ {include = "gguf/py.typed"},
35
+ ]
36
+
37
+ [tool.poetry.dependencies]
38
+ python = ">=3.10"
39
+
40
+ [tool.poetry.group.dev.dependencies]
41
+ pytest = "^5.2"
42
+
43
+ [build-system]
44
+ requires = ["poetry-core>=1.0.0"]
45
+ build-backend = "poetry.core.masonry.api"
gguf-py/tests/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ from .test_metadata import *
gguf-py/tests/test_gguf_reader_validation.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import struct
2
+ import numpy as np
3
+ import pytest
4
+
5
+ from gguf.gguf_reader import GGUFReader
6
+
7
+
8
+ def _write_gguf(path, n_dims_field, dims):
9
+ buf = b'GGUF' + struct.pack('<IQQ', 3, 1, 0) # version 3, 1 tensor, 0 kv
10
+ name = b'bad_tensor'
11
+ buf += struct.pack('<Q', len(name)) + name
12
+ buf += struct.pack('<I', n_dims_field)
13
+ for d in dims:
14
+ buf += struct.pack('<Q', d)
15
+ buf += struct.pack('<I', 0) # dtype F32
16
+ buf += struct.pack('<Q', 0) # tensor offset
17
+ buf += b'\x00' * 64
18
+ path.write_bytes(buf)
19
+
20
+
21
+ def test_n_dims_upper_bound(tmp_path):
22
+ # crafted file claims 1_000_000 dims; must be rejected, not read past EOF
23
+ p = tmp_path / 'evil_ndims.gguf'
24
+ _write_gguf(p, 1_000_000, [1] * 8)
25
+ with pytest.raises(ValueError, match='exceeds GGML_MAX_DIMS'):
26
+ GGUFReader(p)
27
+
28
+
29
+ def test_dims_product_no_uint64_wraparound(tmp_path):
30
+ # dims whose true product overflows uint64; np.prod would wrap to 4 and
31
+ # silently pass an undersized read. The reader must not accept it.
32
+ dims = [4194305, 4194305, 211106198978564]
33
+ assert int(np.prod(np.array(dims, dtype=np.uint64))) == 4 # the wrap bug
34
+ p = tmp_path / 'evil_overflow.gguf'
35
+ _write_gguf(p, len(dims), dims)
36
+ with pytest.raises(ValueError):
37
+ GGUFReader(p)
gguf-py/tests/test_metadata.py ADDED
@@ -0,0 +1,238 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+
3
+ import unittest
4
+ from pathlib import Path
5
+ import os
6
+ import sys
7
+
8
+ # Necessary to load the local gguf package
9
+ if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists():
10
+ sys.path.insert(0, str(Path(__file__).parent.parent))
11
+
12
+ import gguf
13
+
14
+
15
+ class TestMetadataMethod(unittest.TestCase):
16
+
17
+ def test_id_to_title(self):
18
+ self.assertEqual(gguf.Metadata.id_to_title("Mixtral-8x7B-Instruct-v0.1"), "Mixtral 8x7B Instruct v0.1")
19
+ self.assertEqual(gguf.Metadata.id_to_title("Meta-Llama-3-8B"), "Meta Llama 3 8B")
20
+ self.assertEqual(gguf.Metadata.id_to_title("hermes-2-pro-llama-3-8b-DPO"), "Hermes 2 Pro Llama 3 8b DPO")
21
+
22
+ def test_get_model_id_components(self):
23
+ # This is the basic standard form with organization marker
24
+ self.assertEqual(gguf.Metadata.get_model_id_components("Mistral/Mixtral-8x7B-Instruct-v0.1"),
25
+ ('Mixtral-8x7B-Instruct-v0.1', "Mistral", 'Mixtral', 'Instruct', 'v0.1', '8x7B'))
26
+
27
+ # Similar to basic standard form but without organization marker
28
+ self.assertEqual(gguf.Metadata.get_model_id_components("Mixtral-8x7B-Instruct-v0.1"),
29
+ ('Mixtral-8x7B-Instruct-v0.1', None, 'Mixtral', 'Instruct', 'v0.1', '8x7B'))
30
+
31
+ # Missing version
32
+ self.assertEqual(gguf.Metadata.get_model_id_components("Mixtral-8x7B-Instruct"),
33
+ ('Mixtral-8x7B-Instruct', None, 'Mixtral', 'Instruct', None, '8x7B'))
34
+
35
+ # Missing finetune
36
+ self.assertEqual(gguf.Metadata.get_model_id_components("Mixtral-8x7B-v0.1"),
37
+ ('Mixtral-8x7B-v0.1', None, 'Mixtral', None, 'v0.1', '8x7B'))
38
+
39
+ # Base name and size label only
40
+ self.assertEqual(gguf.Metadata.get_model_id_components("Mixtral-8x7B"),
41
+ ('Mixtral-8x7B', None, 'Mixtral', None, None, '8x7B'))
42
+
43
+ # Base name and version only
44
+ self.assertEqual(gguf.Metadata.get_model_id_components("Mixtral-v0.1"),
45
+ ('Mixtral-v0.1', None, 'Mixtral', None, 'v0.1', None))
46
+
47
+ ## Edge Cases ##
48
+
49
+ # This is too ambiguous... best to err on caution and output nothing
50
+ self.assertEqual(gguf.Metadata.get_model_id_components("Mixtral"),
51
+ ('Mixtral', None, None, None, None, None))
52
+
53
+ # Basename has numbers mixed in and also size label provided. Must avoid capturing number in basename
54
+ self.assertEqual(gguf.Metadata.get_model_id_components("NousResearch/Meta-Llama-3-8B"),
55
+ ('Meta-Llama-3-8B', "NousResearch", 'Meta-Llama-3', None, None, '8B'))
56
+
57
+ # Non standard naming
58
+ self.assertEqual(gguf.Metadata.get_model_id_components("Qwen1.5-MoE-A2.7B-Chat"),
59
+ ('Qwen1.5-MoE-A2.7B-Chat', None, 'Qwen1.5-MoE', 'Chat', None, 'A2.7B'))
60
+
61
+ # Capture 'sub size labels' e.g. A14B in '57B-A14B' usually refers to activated params/weight count
62
+ self.assertEqual(gguf.Metadata.get_model_id_components("Qwen2-57B-A14B-Instruct"),
63
+ ('Qwen2-57B-A14B-Instruct', None, 'Qwen2', 'Instruct', None, '57B-A14B'))
64
+
65
+ # Check that it can handle a real model id with no version code
66
+ # Note that 4k in this string is non standard and microsoft were referring to context length rather than weight count
67
+ self.assertEqual(gguf.Metadata.get_model_id_components("microsoft/Phi-3-mini-4k-instruct", 4 * 10**9),
68
+ ('Phi-3-mini-4k-instruct', 'microsoft', 'Phi-3', '4k-instruct', None, 'mini'))
69
+
70
+ # There is some legitimate models with only thousands of parameters
71
+ self.assertEqual(gguf.Metadata.get_model_id_components("delphi-suite/stories-llama2-50k", 50 * 10**3),
72
+ ('stories-llama2-50k', 'delphi-suite', 'stories-llama2', None, None, '50K'))
73
+
74
+ # Non standard and not easy to disambiguate
75
+ self.assertEqual(gguf.Metadata.get_model_id_components("DeepSeek-Coder-V2-Lite-Instruct"),
76
+ ('DeepSeek-Coder-V2-Lite-Instruct', None, 'DeepSeek-Coder-V2-Lite', 'Instruct', None, None))
77
+
78
+ # This is a real model_id where they append 2DPO to refer to Direct Preference Optimization
79
+ self.assertEqual(gguf.Metadata.get_model_id_components("crestf411/daybreak-kunoichi-2dpo-7b"),
80
+ ('daybreak-kunoichi-2dpo-7b', 'crestf411', 'daybreak-kunoichi', '2dpo', None, '7B'))
81
+
82
+ # This is a real model id where the weight size has a decimal point
83
+ self.assertEqual(gguf.Metadata.get_model_id_components("Qwen2-0.5B-Instruct"),
84
+ ('Qwen2-0.5B-Instruct', None, 'Qwen2', 'Instruct', None, '0.5B'))
85
+
86
+ # Uses an underscore in the size label
87
+ self.assertEqual(gguf.Metadata.get_model_id_components("smallcloudai/Refact-1_6B-fim"),
88
+ ('Refact-1_6B-fim', 'smallcloudai', 'Refact', 'fim', None, '1.6B'))
89
+
90
+ # Uses Iter3 for the version
91
+ self.assertEqual(gguf.Metadata.get_model_id_components("UCLA-AGI/Gemma-2-9B-It-SPPO-Iter3"),
92
+ ('Gemma-2-9B-It-SPPO-Iter3', 'UCLA-AGI', 'Gemma-2', 'It-SPPO', 'Iter3', '9B'))
93
+
94
+ # Has two potential versions in the basename
95
+ self.assertEqual(gguf.Metadata.get_model_id_components("NousResearch/Hermes-2-Theta-Llama-3-8B"),
96
+ ('Hermes-2-Theta-Llama-3-8B', 'NousResearch', 'Hermes-2-Theta-Llama-3', None, None, '8B'))
97
+
98
+ # Potential version in the basename
99
+ self.assertEqual(gguf.Metadata.get_model_id_components("SeaLLMs/SeaLLMs-v3-7B-Chat"),
100
+ ('SeaLLMs-v3-7B-Chat', 'SeaLLMs', 'SeaLLMs-v3', 'Chat', None, '7B'))
101
+
102
+ # Underscore in the basename, and 1m for the context size
103
+ self.assertEqual(gguf.Metadata.get_model_id_components("internlm/internlm2_5-7b-chat-1m", 7 * 10**9),
104
+ ('internlm2_5-7b-chat-1m', 'internlm', 'internlm2_5', 'chat-1m', None, '7B'))
105
+
106
+ # Version before the finetune name
107
+ self.assertEqual(gguf.Metadata.get_model_id_components("pszemraj/jamba-900M-v0.13-KIx2"),
108
+ ('jamba-900M-v0.13-KIx2', 'pszemraj', 'jamba', 'KIx2', 'v0.13', '900M'))
109
+
110
+ # TODO: hf suffix which could be ignored but isn't
111
+ self.assertEqual(gguf.Metadata.get_model_id_components("state-spaces/mamba-2.8b-hf"),
112
+ ('mamba-2.8b-hf', 'state-spaces', 'mamba', 'hf', None, '2.8B'))
113
+
114
+ # Two sizes, don't merge them, the other is the number of tokens on which it was trained
115
+ self.assertEqual(gguf.Metadata.get_model_id_components("abacaj/llama-161M-100B", 161 * 10**6),
116
+ ('llama-161M-100B', 'abacaj', 'llama', '100b', None, '161M'))
117
+
118
+ # It's a trap, there is no size label
119
+ self.assertEqual(gguf.Metadata.get_model_id_components("SparseLLM/relu-100B", 1340 * 10**6),
120
+ ('relu-100B', 'SparseLLM', 'relu', '100b', None, None))
121
+
122
+ # Weird size notation
123
+ self.assertEqual(gguf.Metadata.get_model_id_components("bigscience/bloom-7b1-petals"),
124
+ ('bloom-7b1-petals', 'bigscience', 'bloom', 'petals', None, '7.1B'))
125
+
126
+ # Ignore full-text size labels when there are number-based ones, and deduplicate size labels
127
+ self.assertEqual(gguf.Metadata.get_model_id_components("MaziyarPanahi/GreenNode-mini-7B-multilingual-v1olet-Mistral-7B-Instruct-v0.1"),
128
+ ('GreenNode-mini-7B-multilingual-v1olet-Mistral-7B-Instruct-v0.1', 'MaziyarPanahi', 'GreenNode-mini', 'multilingual-v1olet-Mistral-Instruct', 'v0.1', '7B'))
129
+
130
+ # Instruct in a name without a size label
131
+ self.assertEqual(gguf.Metadata.get_model_id_components("mistralai/Mistral-Nemo-Instruct-2407"),
132
+ ('Mistral-Nemo-Instruct-2407', 'mistralai', 'Mistral-Nemo', 'Instruct', '2407', None))
133
+
134
+ # Non-obvious splitting relying on 'chat' keyword
135
+ self.assertEqual(gguf.Metadata.get_model_id_components("deepseek-ai/DeepSeek-V2-Chat-0628"),
136
+ ('DeepSeek-V2-Chat-0628', 'deepseek-ai', 'DeepSeek-V2', 'Chat', '0628', None))
137
+
138
+ # Multiple versions
139
+ self.assertEqual(gguf.Metadata.get_model_id_components("OpenGVLab/Mini-InternVL-Chat-2B-V1-5"),
140
+ ('Mini-InternVL-Chat-2B-V1-5', 'OpenGVLab', 'Mini-InternVL', 'Chat', 'V1-5', '2B'))
141
+
142
+ # TODO: DPO in the name
143
+ self.assertEqual(gguf.Metadata.get_model_id_components("jondurbin/bagel-dpo-2.8b-v0.2"),
144
+ ('bagel-dpo-2.8b-v0.2', 'jondurbin', 'bagel-dpo', None, 'v0.2', '2.8B'))
145
+
146
+ # DPO in name, but can't be used for the finetune to keep 'LLaMA-3' in the basename
147
+ self.assertEqual(gguf.Metadata.get_model_id_components("voxmenthe/SFR-Iterative-DPO-LLaMA-3-8B-R-unquantized"),
148
+ ('SFR-Iterative-DPO-LLaMA-3-8B-R-unquantized', 'voxmenthe', 'SFR-Iterative-DPO-LLaMA-3', 'R-unquantized', None, '8B'))
149
+
150
+ # Too ambiguous
151
+ # TODO: should "base" be a 'finetune' or 'size_label'?
152
+ # (in this case it should be a size label, but other models use it to signal that they are not finetuned)
153
+ self.assertEqual(gguf.Metadata.get_model_id_components("microsoft/Florence-2-base"),
154
+ ('Florence-2-base', 'microsoft', None, None, None, None))
155
+
156
+ ## Invalid cases ##
157
+
158
+ # Start with a dash and has dashes in rows
159
+ self.assertEqual(gguf.Metadata.get_model_id_components("mistralai/-Mistral--Nemo-Base-2407-"),
160
+ ('-Mistral--Nemo-Base-2407-', 'mistralai', 'Mistral-Nemo-Base', None, '2407', None))
161
+
162
+ ## LoRA ##
163
+
164
+ self.assertEqual(gguf.Metadata.get_model_id_components("Llama-3-Instruct-abliteration-LoRA-8B"),
165
+ ('Llama-3-Instruct-abliteration-LoRA-8B', None, 'Llama-3', 'Instruct-abliteration-LoRA', None, '8B'))
166
+
167
+ # Negative size --> output is a LoRA adapter --> prune "LoRA" out of the name to avoid redundancy with the suffix
168
+ self.assertEqual(gguf.Metadata.get_model_id_components("Llama-3-Instruct-abliteration-LoRA-8B", -1234),
169
+ ('Llama-3-Instruct-abliteration-LoRA-8B', None, 'Llama-3', 'Instruct-abliteration', None, '8B'))
170
+
171
+ def test_apply_metadata_heuristic_from_model_card(self):
172
+ model_card = {
173
+ 'tags': ['Llama-3', 'instruct', 'finetune', 'chatml', 'DPO', 'RLHF', 'gpt4', 'synthetic data', 'distillation', 'function calling', 'json mode', 'axolotl'],
174
+ 'model-index': [{'name': 'Mixtral-8x7B-Instruct-v0.1', 'results': []}],
175
+ 'language': ['en'],
176
+ 'datasets': ['teknium/OpenHermes-2.5'],
177
+ 'widget': [{'example_title': 'Hermes 2 Pro', 'messages': [{'role': 'system', 'content': 'You are a sentient, superintelligent artificial general intelligence, here to teach and assist me.'}, {'role': 'user', 'content': 'Write a short story about Goku discovering kirby has teamed up with Majin Buu to destroy the world.'}]}],
178
+ 'base_model': ["EmbeddedLLM/Mistral-7B-Merge-14-v0", "janai-hq/trinity-v1"]
179
+ }
180
+ got = gguf.Metadata.apply_metadata_heuristic(gguf.Metadata(), model_card, None, None)
181
+ expect = gguf.Metadata()
182
+ expect.base_models=[{'name': 'Mistral 7B Merge 14 v0', 'organization': 'EmbeddedLLM', 'version': '14-v0', 'repo_url': 'https://huggingface.co/EmbeddedLLM/Mistral-7B-Merge-14-v0'}, {'name': 'Trinity v1', 'organization': 'Janai Hq', 'version': 'v1', 'repo_url': 'https://huggingface.co/janai-hq/trinity-v1'}]
183
+ expect.tags=['Llama-3', 'instruct', 'finetune', 'chatml', 'DPO', 'RLHF', 'gpt4', 'synthetic data', 'distillation', 'function calling', 'json mode', 'axolotl']
184
+ expect.languages=['en']
185
+ expect.datasets=[{'name': 'OpenHermes 2.5', 'organization': 'Teknium', 'version': '2.5', 'repo_url': 'https://huggingface.co/teknium/OpenHermes-2.5'}]
186
+ self.assertEqual(got, expect)
187
+
188
+ # Base Model spec is inferred from model id
189
+ model_card = {'base_models': 'teknium/OpenHermes-2.5'}
190
+ expect = gguf.Metadata(base_models=[{'name': 'OpenHermes 2.5', 'organization': 'Teknium', 'version': '2.5', 'repo_url': 'https://huggingface.co/teknium/OpenHermes-2.5'}])
191
+ got = gguf.Metadata.apply_metadata_heuristic(gguf.Metadata(), model_card, None, None)
192
+ self.assertEqual(got, expect)
193
+
194
+ # Base Model spec is only url
195
+ model_card = {'base_models': ['https://huggingface.co/teknium/OpenHermes-2.5']}
196
+ expect = gguf.Metadata(base_models=[{'name': 'OpenHermes 2.5', 'organization': 'Teknium', 'version': '2.5', 'repo_url': 'https://huggingface.co/teknium/OpenHermes-2.5'}])
197
+ got = gguf.Metadata.apply_metadata_heuristic(gguf.Metadata(), model_card, None, None)
198
+ self.assertEqual(got, expect)
199
+
200
+ # Base Model spec is given directly
201
+ model_card = {'base_models': [{'name': 'OpenHermes 2.5', 'organization': 'Teknium', 'version': '2.5', 'repo_url': 'https://huggingface.co/teknium/OpenHermes-2.5'}]}
202
+ expect = gguf.Metadata(base_models=[{'name': 'OpenHermes 2.5', 'organization': 'Teknium', 'version': '2.5', 'repo_url': 'https://huggingface.co/teknium/OpenHermes-2.5'}])
203
+ got = gguf.Metadata.apply_metadata_heuristic(gguf.Metadata(), model_card, None, None)
204
+ self.assertEqual(got, expect)
205
+
206
+ # Dataset spec is inferred from model id
207
+ model_card = {'datasets': 'teknium/OpenHermes-2.5'}
208
+ expect = gguf.Metadata(datasets=[{'name': 'OpenHermes 2.5', 'organization': 'Teknium', 'version': '2.5', 'repo_url': 'https://huggingface.co/teknium/OpenHermes-2.5'}])
209
+ got = gguf.Metadata.apply_metadata_heuristic(gguf.Metadata(), model_card, None, None)
210
+ self.assertEqual(got, expect)
211
+
212
+ # Dataset spec is only url
213
+ model_card = {'datasets': ['https://huggingface.co/teknium/OpenHermes-2.5']}
214
+ expect = gguf.Metadata(datasets=[{'name': 'OpenHermes 2.5', 'organization': 'Teknium', 'version': '2.5', 'repo_url': 'https://huggingface.co/teknium/OpenHermes-2.5'}])
215
+ got = gguf.Metadata.apply_metadata_heuristic(gguf.Metadata(), model_card, None, None)
216
+ self.assertEqual(got, expect)
217
+
218
+ # Dataset spec is given directly
219
+ model_card = {'datasets': [{'name': 'OpenHermes 2.5', 'organization': 'Teknium', 'version': '2.5', 'repo_url': 'https://huggingface.co/teknium/OpenHermes-2.5'}]}
220
+ expect = gguf.Metadata(datasets=[{'name': 'OpenHermes 2.5', 'organization': 'Teknium', 'version': '2.5', 'repo_url': 'https://huggingface.co/teknium/OpenHermes-2.5'}])
221
+ got = gguf.Metadata.apply_metadata_heuristic(gguf.Metadata(), model_card, None, None)
222
+ self.assertEqual(got, expect)
223
+
224
+ def test_apply_metadata_heuristic_from_hf_parameters(self):
225
+ hf_params = {"_name_or_path": "./hermes-2-pro-llama-3-8b-DPO"}
226
+ got = gguf.Metadata.apply_metadata_heuristic(gguf.Metadata(), model_card=None, hf_params=hf_params, model_path=None)
227
+ expect = gguf.Metadata(name='Hermes 2 Pro Llama 3 8b DPO', finetune='DPO', basename='hermes-2-pro-llama-3', size_label='8B')
228
+ self.assertEqual(got, expect)
229
+
230
+ def test_apply_metadata_heuristic_from_model_dir(self):
231
+ model_dir_path = Path("./hermes-2-pro-llama-3-8b-DPO")
232
+ got = gguf.Metadata.apply_metadata_heuristic(gguf.Metadata(), model_card=None, hf_params=None, model_path=model_dir_path)
233
+ expect = gguf.Metadata(name='Hermes 2 Pro Llama 3 8b DPO', finetune='DPO', basename='hermes-2-pro-llama-3', size_label='8B')
234
+ self.assertEqual(got, expect)
235
+
236
+
237
+ if __name__ == "__main__":
238
+ unittest.main()
gguf-py/tests/test_quants.py ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+
3
+ # Test gguf.quants so that it exactly matches the C implementation of the (de)quantization
4
+
5
+ # NOTE: this is kind of a mess, but at least it worked for initially testing the Python implementations.
6
+
7
+ from __future__ import annotations
8
+
9
+ import argparse
10
+ from math import prod
11
+ import os
12
+ import sys
13
+ from pathlib import Path
14
+ import ctypes
15
+ import logging
16
+ import numpy as np
17
+
18
+ # Necessary to load the local gguf package
19
+ if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / 'gguf-py').exists():
20
+ sys.path.insert(0, str(Path(__file__).parent.parent))
21
+
22
+ import gguf
23
+ from gguf.constants import GGMLQuantizationType
24
+
25
+
26
+ logger = logging.getLogger("test-quants")
27
+
28
+
29
+ c_float_p = ctypes.POINTER(ctypes.c_float)
30
+
31
+
32
+ class ggml_init_params(ctypes.Structure):
33
+ _fields_ = [
34
+ ("mem_size", ctypes.c_size_t),
35
+ ("mem_buffer", ctypes.c_void_p),
36
+ ("no_alloc", ctypes.c_bool),
37
+ ]
38
+
39
+
40
+ class GGMLQuants:
41
+ libggml: ctypes.CDLL
42
+
43
+ def __init__(self, libggml: Path):
44
+ self.libggml = ctypes.CDLL(str(libggml))
45
+ self.libggml.ggml_quantize_chunk.restype = ctypes.c_size_t
46
+ # enum ggml_type type,
47
+ # const float * src,
48
+ # void * dst,
49
+ # int64_t start,
50
+ # int64_t nrows,
51
+ # int64_t n_per_row,
52
+ # const float * imatrix) {
53
+ self.libggml.ggml_quantize_chunk.argtypes = (
54
+ ctypes.c_int,
55
+ ctypes.POINTER(ctypes.c_float),
56
+ ctypes.c_void_p,
57
+ ctypes.c_int64,
58
+ ctypes.c_int64,
59
+ ctypes.c_int64,
60
+ ctypes.POINTER(ctypes.c_float),
61
+ )
62
+
63
+ self.libggml.ggml_quantize_requires_imatrix.restype = ctypes.c_bool
64
+ self.libggml.ggml_quantize_requires_imatrix.argtypes = (ctypes.c_int,)
65
+
66
+ for t in (
67
+ "q4_0", "q4_1", "q5_0", "q5_1", "q8_0",
68
+ "q2_K", "q3_K", "q4_K", "q5_K", "q6_K",
69
+ "tq1_0", "tq2_0",
70
+ "mxfp4",
71
+ "nvfp4",
72
+ "iq2_xxs", "iq2_xs", "iq2_s", "iq3_xxs", "iq3_s", "iq1_s", "iq1_m",
73
+ "iq4_nl", "iq4_xs",
74
+ ):
75
+ dequant_func: ctypes._NamedFuncPointer = getattr(self.libggml, "dequantize_row_" + t)
76
+ dequant_func.restype = None
77
+ dequant_func.argtypes = (ctypes.c_void_p, ctypes.POINTER(ctypes.c_float), ctypes.c_int64)
78
+
79
+ self.libggml.ggml_fp16_to_fp32_row.restype = None
80
+ self.libggml.ggml_fp16_to_fp32_row.argtypes = (ctypes.POINTER(ctypes.c_uint16), ctypes.POINTER(ctypes.c_float), ctypes.c_int64)
81
+ self.libggml.ggml_bf16_to_fp32_row.restype = None
82
+ self.libggml.ggml_bf16_to_fp32_row.argtypes = (ctypes.POINTER(ctypes.c_uint16), ctypes.POINTER(ctypes.c_float), ctypes.c_int64)
83
+
84
+ self.libggml.ggml_init.argtypes = (ggml_init_params,)
85
+
86
+ self.libggml.ggml_init(ggml_init_params(1 * 1024 * 1024, 0, False))
87
+
88
+ def dequantize(self, tensor: np.ndarray, qtype: GGMLQuantizationType) -> np.ndarray:
89
+ result = np.zeros(gguf.quant_shape_from_byte_shape(tensor.shape, qtype), dtype=np.float32, order="C")
90
+ if qtype == GGMLQuantizationType.F32:
91
+ # no-op
92
+ result = tensor.view(np.float32)
93
+ elif qtype == GGMLQuantizationType.F16:
94
+ self.libggml.ggml_fp16_to_fp32_row(tensor.ctypes.data_as(ctypes.POINTER(ctypes.c_uint16)), result.ctypes.data_as(c_float_p), result.size)
95
+ elif qtype == GGMLQuantizationType.BF16:
96
+ self.libggml.ggml_bf16_to_fp32_row(tensor.ctypes.data_as(ctypes.POINTER(ctypes.c_uint16)), result.ctypes.data_as(c_float_p), result.size)
97
+ else:
98
+ lw_qname = qtype.name.lower()
99
+ if lw_qname[-1] == "k":
100
+ lw_qname = lw_qname[:-1] + "K"
101
+ dequant_func: ctypes._NamedFuncPointer = getattr(self.libggml, "dequantize_row_" + lw_qname)
102
+ dequant_func(tensor.ctypes.data_as(ctypes.c_void_p), result.ctypes.data_as(c_float_p), result.size)
103
+ return result
104
+
105
+ def quantize(self, data: np.ndarray, qtype: GGMLQuantizationType) -> np.ndarray:
106
+ result = np.zeros(gguf.quant_shape_to_byte_shape(data.shape, qtype), dtype=np.uint8, order="C")
107
+ if self.libggml.ggml_quantize_requires_imatrix(qtype.value):
108
+ # TODO: is a column-wise sum of squares appropriate?
109
+ qw = np.sum((data * data).reshape((-1, data.shape[-1])), axis=0).ctypes.data_as(c_float_p)
110
+ else:
111
+ qw = ctypes.cast(0, c_float_p)
112
+ result_size = self.libggml.ggml_quantize_chunk(qtype.value, data.ctypes.data_as(c_float_p), result.ctypes.data_as(ctypes.c_void_p), 0, prod(data.shape[:-1]), data.shape[-1], qw)
113
+ assert result.size == result_size
114
+ return result
115
+
116
+
117
+ def compare_tensors(t1: np.ndarray, t2: np.ndarray, qtype: GGMLQuantizationType) -> bool:
118
+ same = np.array_equal(t1, t2)
119
+ if same:
120
+ return True
121
+ else:
122
+ block_size, type_size = gguf.GGML_QUANT_SIZES[qtype]
123
+ if t1.dtype == np.float32:
124
+ t1 = t1.reshape((-1, block_size))
125
+ t2 = t2.reshape((-1, block_size))
126
+ else:
127
+ t1 = t1.reshape((-1, type_size))
128
+ t2 = t2.reshape((-1, type_size))
129
+ x = t1.view(np.uint8) ^ t2.view(np.uint8)
130
+ diff_bits = np.count_nonzero(np.unpackbits(x, axis=-1), axis=-1)
131
+ num_bad_blocks = np.count_nonzero(diff_bits, axis=0)
132
+ if num_bad_blocks == 0 and t1.shape == t2.shape:
133
+ logger.debug("Bits are equal, but arrays don't match, likely contains NANs")
134
+ return True
135
+ logger.debug(f"{num_bad_blocks} bad blocks ({100 * num_bad_blocks / x.shape[0]:.6f}%)")
136
+ bad_block_id = np.argmax(diff_bits, axis=0)
137
+ logger.debug(f"Worst block id: {bad_block_id}")
138
+ logger.debug(f"Sample bad block ({diff_bits[bad_block_id]} differing bits):\n{t1[bad_block_id]}\nReference:\n{t2[bad_block_id]}")
139
+
140
+ sum_diff_bits = np.sum(diff_bits)
141
+ logger.debug(f"{sum_diff_bits} bits differ ({100 * sum_diff_bits / (x.size * 8):.6f}%)")
142
+ return False
143
+
144
+
145
+ def do_test(libggml_path: Path, quick: bool = False, user_type: GGMLQuantizationType | None = None):
146
+ ggml_quants = GGMLQuants(libggml_path)
147
+
148
+ np.set_printoptions(precision=None, threshold=(4 * 256) + 1, formatter={"int": lambda n: "0x%02X" % n})
149
+
150
+ r = np.random.randn(8, 1024, 1024).astype(np.float32, copy=False)
151
+ # test zero blocks
152
+ r[0, 0, :] = 0
153
+ ## Maybe test infinities? (can make NANs, not really useful in practice)
154
+ # r[0, 1, 0] = np.inf
155
+ # r[0, 2, 0] = -np.inf
156
+ # r[0, 3, 0] = np.inf
157
+ # r[0, 3, 1] = -np.inf
158
+
159
+ for qtype in ((GGMLQuantizationType.F16, *gguf.quants._type_traits.keys()) if user_type is None else (user_type,)):
160
+ has_dequantize = False
161
+ has_quantize = False
162
+
163
+ try:
164
+ gguf.dequantize(np.zeros((gguf.GGML_QUANT_SIZES[qtype][1]), dtype=np.uint8), qtype)
165
+ has_dequantize = True
166
+ except (NotImplementedError, AssertionError) as e:
167
+ if isinstance(e, AssertionError):
168
+ logger.error(f"Error with {qtype.name}: {e}")
169
+ raise e
170
+ try:
171
+ gguf.quantize(np.zeros((gguf.GGML_QUANT_SIZES[qtype][0]), dtype=np.float32), qtype)
172
+ has_quantize = True
173
+ except (NotImplementedError, AssertionError) as e:
174
+ if isinstance(e, AssertionError):
175
+ logger.error(f"Error with {qtype.name}: {e}")
176
+ raise e
177
+
178
+ if not has_dequantize and not has_quantize:
179
+ continue
180
+
181
+ logger.info(f"Testing {qtype.name}")
182
+
183
+ rc = r.copy(order="C")
184
+
185
+ pyq = None
186
+ ggq = None
187
+
188
+ if has_quantize:
189
+ logger.debug(f"Quantizing to {qtype.name} with Python")
190
+ pyq = gguf.quants.quantize(rc, qtype)
191
+
192
+ logger.debug(f"Quantizing to {qtype.name} with C")
193
+ ggq = ggml_quants.quantize(rc, qtype)
194
+
195
+ if qtype == GGMLQuantizationType.F16:
196
+ pyq = pyq.view(np.uint8)
197
+ quant_equal = compare_tensors(pyq, ggq, qtype)
198
+
199
+ if not quant_equal:
200
+ logger.error(f"Quantization to {qtype.name} does not match ❌")
201
+ else:
202
+ logger.info(f"Quantization to {qtype.name} matches exactly ✅")
203
+
204
+ if has_dequantize:
205
+ if ggq is None and not quick:
206
+ logger.debug(f"Quantizing to {qtype.name} with C")
207
+ ggq = ggml_quants.quantize(rc, qtype)
208
+
209
+ if ggq is not None:
210
+ logger.debug(f"Dequantizing from {qtype.name} with Python")
211
+ pydq = gguf.quants.dequantize(ggq, qtype)
212
+ logger.debug(f"Dequantizing from {qtype.name} with C")
213
+ ggdq = ggml_quants.dequantize(ggq, qtype)
214
+
215
+ dequant_equal = compare_tensors(pydq, ggdq, qtype)
216
+
217
+ if not dequant_equal:
218
+ logger.error(f"Dequantization from {qtype.name} does not match ❌")
219
+ else:
220
+ logger.info(f"Dequantization from {qtype.name} matches exactly ✅")
221
+
222
+ rq_shape = gguf.quants.quant_shape_to_byte_shape((8, 1024, 1024 // 2), qtype)
223
+ rq = np.random.random(rq_shape).astype(np.float16).view(np.uint8)
224
+
225
+ logger.debug(f"Dequantizing random f16 data as {qtype.name} with Python")
226
+ pydq = gguf.quants.dequantize(rq, qtype)
227
+ logger.debug(f"Dequantizing random f16 data as {qtype.name} with C")
228
+ ggdq = ggml_quants.dequantize(rq, qtype)
229
+
230
+ dequant_equal = compare_tensors(pydq, ggdq, qtype)
231
+
232
+ if not dequant_equal:
233
+ logger.error(f"Dequantization from random f16 data as {qtype.name} does not match ❌")
234
+ else:
235
+ logger.info(f"Dequantization from random f16 data as {qtype.name} matches exactly ✅")
236
+
237
+
238
+ if __name__ == "__main__":
239
+ parser = argparse.ArgumentParser(description="Test Python (de)quantization against the reference C implementation")
240
+ parser.add_argument("--libggml", type=Path, default=Path(__file__).parent.parent.parent / "build" / "bin" / "libggml.so", help="The path to libggml.so")
241
+ parser.add_argument("--quick", action="store_true", help="Don't quantize with C when it's not strictly necessary")
242
+ parser.add_argument("--type", type=str, help="The quant type to test (all by default)")
243
+
244
+ args = parser.parse_args()
245
+
246
+ logging.basicConfig(level=logging.DEBUG)
247
+
248
+ do_test(args.libggml, args.quick, GGMLQuantizationType[args.type.upper()] if args.type is not None else None)
gpu_sample.csv ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 0, 0, 65.43, 1710
2
+ 0, 0, 65.63, 1710
3
+ 0, 0, 65.54, 1710
4
+ 0, 0, 65.32, 1710
5
+ 0, 0, 65.44, 1710
6
+ 0, 0, 65.56, 1710
7
+ 0, 0, 65.40, 1710
8
+ 0, 0, 65.35, 1710
9
+ 0, 0, 65.41, 1710
10
+ 0, 0, 65.49, 1710
11
+ 94, 3, 65.92, 1710
12
+ 1, 0, 68.00, 1710
13
+ 0, 0, 71.85, 1710
14
+ 95, 7, 98.81, 1710
15
+ 95, 7, 110.95, 1710
16
+ 95, 7, 111.60, 1710
17
+ 95, 7, 111.60, 1710
18
+ 95, 8, 111.86, 1710
19
+ 95, 7, 111.72, 1710
20
+ 95, 7, 111.55, 1710
21
+ 95, 8, 111.88, 1710
22
+ 95, 7, 112.17, 1710
23
+ 95, 8, 112.02, 1710
24
+ 95, 7, 112.19, 1710
25
+ 95, 7, 111.91, 1710
26
+ 95, 8, 112.34, 1710
27
+ 95, 7, 112.48, 1710
28
+ 95, 7, 112.46, 1710
29
+ 95, 7, 112.67, 1710
30
+ 96, 7, 112.25, 1710
31
+ 95, 7, 112.41, 1710
32
+ 95, 7, 112.70, 1710
33
+ 95, 7, 112.56, 1710
34
+ 41, 4, 109.15, 1710
grammars/README.md ADDED
@@ -0,0 +1,408 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # GBNF Guide
2
+
3
+ GBNF (GGML BNF) is a format for defining [formal grammars](https://en.wikipedia.org/wiki/Formal_grammar) to constrain model outputs in `llama.cpp`. For example, you can use it to force the model to generate valid JSON, or speak only in emojis. GBNF grammars are supported in various ways in `tools/cli`, `tools/completion` and `tools/server`.
4
+
5
+ ## Background
6
+
7
+ [Backus-Naur Form (BNF)](https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form) is a notation for describing the syntax of formal languages like programming languages, file formats, and protocols. GBNF is an extension of BNF that primarily adds a few modern regex-like features.
8
+
9
+ ## Basics
10
+
11
+ In GBNF, we define *production rules* that specify how a *non-terminal* (rule name) can be replaced with sequences of *terminals* (characters, specifically Unicode [code points](https://en.wikipedia.org/wiki/Code_point)) and other non-terminals. The basic format of a production rule is `nonterminal ::= sequence...`.
12
+
13
+ ## Example
14
+
15
+ Before going deeper, let's look at some of the features demonstrated in `grammars/chess.gbnf`, a small chess notation grammar:
16
+ ```
17
+ # `root` specifies the pattern for the overall output
18
+ root ::= (
19
+ # it must start with the characters "1. " followed by a sequence
20
+ # of characters that match the `move` rule, followed by a space, followed
21
+ # by another move, and then a newline
22
+ "1. " move " " move "\n"
23
+
24
+ # it's followed by one or more subsequent moves, numbered with one or two digits
25
+ ([1-9] [0-9]? ". " move " " move "\n")+
26
+ )
27
+
28
+ # `move` is an abstract representation, which can be a pawn, nonpawn, or castle.
29
+ # The `[+#]?` denotes the possibility of checking or mate signs after moves
30
+ move ::= (pawn | nonpawn | castle) [+#]?
31
+
32
+ pawn ::= ...
33
+ nonpawn ::= ...
34
+ castle ::= ...
35
+ ```
36
+
37
+ ## Non-Terminals and Terminals
38
+
39
+ Non-terminal symbols (rule names) stand for a pattern of terminals and other non-terminals. They are required to be a dashed lowercase word, like `move`, `castle`, or `check-mate`.
40
+
41
+ Terminals are actual characters ([code points](https://en.wikipedia.org/wiki/Code_point)). They can be specified as a sequence like `"1"` or `"O-O"` or as ranges like `[1-9]` or `[NBKQR]`.
42
+
43
+ ## Characters and character ranges
44
+
45
+ Terminals support the full range of Unicode. Unicode characters can be specified directly in the grammar, for example `hiragana ::= [ぁ-ゟ]`, or with escapes: 8-bit (`\xXX`), 16-bit (`\uXXXX`) or 32-bit (`\UXXXXXXXX`).
46
+
47
+ Character ranges can be negated with `^`:
48
+ ```
49
+ single-line ::= [^\n]+ "\n"
50
+ ```
51
+
52
+ ## Sequences and Alternatives
53
+
54
+ The order of symbols in a sequence matters. For example, in `"1. " move " " move "\n"`, the `"1. "` must come before the first `move`, etc.
55
+
56
+ Alternatives, denoted by `|`, give different sequences that are acceptable. For example, in `move ::= pawn | nonpawn | castle`, `move` can be a `pawn` move, a `nonpawn` move, or a `castle`.
57
+
58
+ Parentheses `()` can be used to group sequences, which allows for embedding alternatives in a larger rule or applying repetition and optional symbols (below) to a sequence.
59
+
60
+ ## Repetition and Optional Symbols
61
+
62
+ - `*` after a symbol or sequence means that it can be repeated zero or more times (equivalent to `{0,}`).
63
+ - `+` denotes that the symbol or sequence should appear one or more times (equivalent to `{1,}`).
64
+ - `?` makes the preceding symbol or sequence optional (equivalent to `{0,1}`).
65
+ - `{m}` repeats the precedent symbol or sequence exactly `m` times
66
+ - `{m,}` repeats the precedent symbol or sequence at least `m` times
67
+ - `{m,n}` repeats the precedent symbol or sequence at between `m` and `n` times (included)
68
+ - `{0,n}` repeats the precedent symbol or sequence at most `n` times (included)
69
+
70
+ ## Tokens
71
+
72
+ Tokens allow grammars to match specific tokenizer tokens rather than character sequences. This is useful for constraining outputs based on special tokens (like `<think>` or `</think>`).
73
+
74
+ Tokens can be specified in two ways:
75
+
76
+ 1. **Token ID**: Use angle brackets with the token ID in square brackets: `<[token-id]>`. For example, `<[1000]>` matches the token with ID 1000.
77
+
78
+ 2. **Token string**: Use angle brackets with the token text directly: `<token>`. For example, `<think>` will match the token whose text is exactly `<think>`. This only works if the string tokenizes to exactly one token in the vocabulary, otherwise the grammar will fail to parse.
79
+
80
+ You can negate token matches using the `!` prefix: `!<[1000]>` or `!<think>` matches any token *except* the specified one.
81
+
82
+ ```
83
+ # Match a thinking block: <think>...</think>
84
+ # Using token strings (requires these to be single tokens in the vocab)
85
+ root ::= <think> thinking </think> .*
86
+ thinking ::= !</think>*
87
+
88
+ # Equivalent grammar using explicit token IDs
89
+ # Assumes token 1000 = <think>, token 1001 = </think>
90
+ root ::= <[1000]> thinking <[1001]> .*
91
+ thinking ::= !<[1001]>*
92
+ ```
93
+
94
+ ## Comments and newlines
95
+
96
+ Comments can be specified with `#`:
97
+ ```
98
+ # defines optional whitespace
99
+ ws ::= [ \t\n]+
100
+ ```
101
+
102
+ Newlines are allowed between rules and between symbols or sequences nested inside parentheses. Additionally, a newline after an alternate marker `|` will continue the current rule, even outside of parentheses.
103
+
104
+ ## The root rule
105
+
106
+ In a full grammar, the `root` rule always defines the starting point of the grammar. In other words, it specifies what the entire output must match.
107
+
108
+ ```
109
+ # a grammar for lists
110
+ root ::= ("- " item)+
111
+ item ::= [^\n]+ "\n"
112
+ ```
113
+
114
+ ## Next steps
115
+
116
+ This guide provides a brief overview. Check out the GBNF files in this directory (`grammars/`) for examples of full grammars. You can try them out with:
117
+ ```
118
+ ./llama-cli -m <model> --grammar-file grammars/some-grammar.gbnf -p 'Some prompt'
119
+ ```
120
+
121
+ `llama.cpp` can also convert JSON schemas to grammars either ahead of time or at each request, see below.
122
+
123
+ ## Troubleshooting
124
+
125
+ Grammars currently have performance gotchas (see https://github.com/ggml-org/llama.cpp/issues/4218).
126
+
127
+ ### Efficient optional repetitions
128
+
129
+ A common pattern is to allow repetitions of a pattern `x` up to N times.
130
+
131
+ While semantically correct, the syntax `x? x? x?.... x?` (with N repetitions) may result in extremely slow sampling. Instead, you can write `x{0,N}` (or `(x (x (x ... (x)?...)?)?)?` w/ N-deep nesting in earlier llama.cpp versions).
132
+
133
+ ## Using GBNF grammars
134
+
135
+ You can use GBNF grammars:
136
+
137
+ - In [llama-server](../tools/server)'s completion endpoints, passed as the `grammar` body field
138
+ - In [llama-cli](../tools/cli) and [llama-completion](../tools/completion), passed as the `--grammar` & `--grammar-file` flags
139
+ - With [test-gbnf-validator](../tests/test-gbnf-validator.cpp), to test them against strings.
140
+
141
+ ## JSON Schemas → GBNF
142
+
143
+ `llama.cpp` supports converting a subset of https://json-schema.org/ to GBNF grammars:
144
+
145
+ - In [llama-server](../tools/server):
146
+ - For any completion endpoints, passed as the `json_schema` body field
147
+ - For the `/chat/completions` endpoint, passed inside the `response_format` body field (e.g. `{"type", "json_object", "schema": {"items": {}}}` or `{ type: "json_schema", json_schema: {"schema": ...} }`)
148
+ - In [llama-cli](../tools/cli) and [llama-completion](../tools/completion), passed as the `--json` / `-j` flag
149
+ - To convert to a grammar ahead of time:
150
+ - in CLI, with [examples/json_schema_to_grammar.py](../examples/json_schema_to_grammar.py)
151
+
152
+ > [!NOTE]
153
+ > The JSON schema is only used to constrain the model output and is not injected into the prompt. The model has no visibility into the schema, so if you want it to understand the expected structure, describe it explicitly in your prompt. This does not apply to tool calling, where schemas are injected into the prompt.
154
+
155
+ Take a look at [tests](../tests/test-json-schema-to-grammar.cpp) to see which features are likely supported (you'll also find usage examples in https://github.com/ggml-org/llama.cpp/pull/5978, https://github.com/ggml-org/llama.cpp/pull/6659 & https://github.com/ggml-org/llama.cpp/pull/6555).
156
+
157
+ ```bash
158
+ llama-cli \
159
+ -hfr bartowski/Phi-3-medium-128k-instruct-GGUF \
160
+ -hff Phi-3-medium-128k-instruct-Q8_0.gguf \
161
+ -j '{
162
+ "type": "array",
163
+ "items": {
164
+ "type": "object",
165
+ "properties": {
166
+ "name": {
167
+ "type": "string",
168
+ "minLength": 1,
169
+ "maxLength": 100
170
+ },
171
+ "age": {
172
+ "type": "integer",
173
+ "minimum": 0,
174
+ "maximum": 150
175
+ }
176
+ },
177
+ "required": ["name", "age"],
178
+ "additionalProperties": false
179
+ },
180
+ "minItems": 10,
181
+ "maxItems": 100
182
+ }' \
183
+ -p 'Generate a {name, age}[] JSON array with famous actors of all ages.'
184
+ ```
185
+
186
+ <details>
187
+
188
+ <summary>Show grammar</summary>
189
+
190
+ You can convert any schema in command-line with:
191
+
192
+ ```bash
193
+ examples/json_schema_to_grammar.py name-age-schema.json
194
+ ```
195
+
196
+ ```
197
+ char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
198
+ item ::= "{" space item-name-kv "," space item-age-kv "}" space
199
+ item-age ::= ([0-9] | ([1-8] [0-9] | [9] [0-9]) | "1" ([0-4] [0-9] | [5] "0")) space
200
+ item-age-kv ::= "\"age\"" space ":" space item-age
201
+ item-name ::= "\"" char{1,100} "\"" space
202
+ item-name-kv ::= "\"name\"" space ":" space item-name
203
+ root ::= "[" space item ("," space item){9,99} "]" space
204
+ space ::= | " " | "\n" [ \t]{0,20}
205
+ ```
206
+
207
+ </details>
208
+
209
+ Here is also a list of known limitations (contributions welcome):
210
+
211
+ - `additionalProperties` defaults to `false` (produces faster grammars + reduces hallucinations).
212
+ - `"additionalProperties": true` may produce keys that contain unescaped newlines.
213
+ - Unsupported features are skipped silently. It is currently advised to use the command-line Python converter (see above) to see any warnings, and to inspect the resulting grammar / test it w/ [llama-gbnf-validator](../examples/gbnf-validator/gbnf-validator.cpp).
214
+ - Can't mix `properties` w/ `anyOf` / `oneOf` in the same type (https://github.com/ggml-org/llama.cpp/issues/7703)
215
+ - [prefixItems](https://json-schema.org/draft/2020-12/json-schema-core#name-prefixitems) is broken (but [items](https://json-schema.org/draft/2020-12/json-schema-core#name-items) works)
216
+ - `minimum`, `exclusiveMinimum`, `maximum`, `exclusiveMaximum`: only supported for `"type": "integer"` for now, not `number`
217
+ - Nested `$ref`s are broken (https://github.com/ggml-org/llama.cpp/issues/8073)
218
+ - [pattern](https://json-schema.org/draft/2020-12/json-schema-validation#name-pattern)s must start with `^` and end with `$`
219
+ - Remote `$ref`s not supported in the C++ version (Python & JavaScript versions fetch https refs)
220
+ - `string` [formats](https://json-schema.org/draft/2020-12/json-schema-validation#name-defined-formats) lack `uri`, `email`
221
+ - No [`patternProperties`](https://json-schema.org/draft/2020-12/json-schema-core#name-patternproperties)
222
+
223
+ And a non-exhaustive list of other unsupported features that are unlikely to be implemented (hard and/or too slow to support w/ stateless grammars):
224
+
225
+ - [`uniqueItems`](https://json-schema.org/draft/2020-12/json-schema-validation#name-uniqueitems)
226
+ - [`contains`](https://json-schema.org/draft/2020-12/json-schema-core#name-contains) / `minContains`
227
+ - `$anchor` (cf. [dereferencing](https://json-schema.org/draft/2020-12/json-schema-core#name-dereferencing))
228
+ - [`not`](https://json-schema.org/draft/2020-12/json-schema-core#name-not)
229
+ - [Conditionals](https://json-schema.org/draft/2020-12/json-schema-core#name-keywords-for-applying-subsche) `if` / `then` / `else` / `dependentSchemas`
230
+
231
+ ### A word about additionalProperties
232
+
233
+ > [!WARNING]
234
+ > The JSON schemas spec states `object`s accept [additional properties](https://json-schema.org/understanding-json-schema/reference/object#additionalproperties) by default.
235
+ > Since this is slow and seems prone to hallucinations, we default to no additional properties.
236
+ > You can set `"additionalProperties": true` in the schema of any object to explicitly allow additional properties.
237
+
238
+ If you're using [Pydantic](https://pydantic.dev/) to generate schemas, you can enable additional properties with the `extra` config on each model class:
239
+
240
+ ```python
241
+ # pip install pydantic
242
+ import json
243
+ from typing import Annotated, List
244
+ from pydantic import BaseModel, Extra, Field
245
+ class QAPair(BaseModel):
246
+ class Config:
247
+ extra = 'allow' # triggers additionalProperties: true in the JSON schema
248
+ question: str
249
+ concise_answer: str
250
+ justification: str
251
+
252
+ class Summary(BaseModel):
253
+ class Config:
254
+ extra = 'allow'
255
+ key_facts: List[Annotated[str, Field(pattern='- .{5,}')]]
256
+ question_answers: List[Annotated[List[QAPair], Field(min_items=5)]]
257
+
258
+ print(json.dumps(Summary.model_json_schema(), indent=2))
259
+ ```
260
+
261
+ <details>
262
+ <summary>Show JSON schema & grammar</summary>
263
+
264
+ ```json
265
+ {
266
+ "$defs": {
267
+ "QAPair": {
268
+ "additionalProperties": true,
269
+ "properties": {
270
+ "question": {
271
+ "title": "Question",
272
+ "type": "string"
273
+ },
274
+ "concise_answer": {
275
+ "title": "Concise Answer",
276
+ "type": "string"
277
+ },
278
+ "justification": {
279
+ "title": "Justification",
280
+ "type": "string"
281
+ }
282
+ },
283
+ "required": [
284
+ "question",
285
+ "concise_answer",
286
+ "justification"
287
+ ],
288
+ "title": "QAPair",
289
+ "type": "object"
290
+ }
291
+ },
292
+ "additionalProperties": true,
293
+ "properties": {
294
+ "key_facts": {
295
+ "items": {
296
+ "pattern": "^- .{5,}$",
297
+ "type": "string"
298
+ },
299
+ "title": "Key Facts",
300
+ "type": "array"
301
+ },
302
+ "question_answers": {
303
+ "items": {
304
+ "items": {
305
+ "$ref": "#/$defs/QAPair"
306
+ },
307
+ "minItems": 5,
308
+ "type": "array"
309
+ },
310
+ "title": "Question Answers",
311
+ "type": "array"
312
+ }
313
+ },
314
+ "required": [
315
+ "key_facts",
316
+ "question_answers"
317
+ ],
318
+ "title": "Summary",
319
+ "type": "object"
320
+ }
321
+ ```
322
+
323
+ ```
324
+ QAPair ::= "{" space QAPair-question-kv "," space QAPair-concise-answer-kv "," space QAPair-justification-kv ( "," space ( QAPair-additional-kv ( "," space QAPair-additional-kv )* ) )? "}" space
325
+ QAPair-additional-k ::= ["] ( [c] ([o] ([n] ([c] ([i] ([s] ([e] ([_] ([a] ([n] ([s] ([w] ([e] ([r] char+ | [^"r] char*) | [^"e] char*) | [^"w] char*) | [^"s] char*) | [^"n] char*) | [^"a] char*) | [^"_] char*) | [^"e] char*) | [^"s] char*) | [^"i] char*) | [^"c] char*) | [^"n] char*) | [^"o] char*) | [j] ([u] ([s] ([t] ([i] ([f] ([i] ([c] ([a] ([t] ([i] ([o] ([n] char+ | [^"n] char*) | [^"o] char*) | [^"i] char*) | [^"t] char*) | [^"a] char*) | [^"c] char*) | [^"i] char*) | [^"f] char*) | [^"i] char*) | [^"t] char*) | [^"s] char*) | [^"u] char*) | [q] ([u] ([e] ([s] ([t] ([i] ([o] ([n] char+ | [^"n] char*) | [^"o] char*) | [^"i] char*) | [^"t] char*) | [^"s] char*) | [^"e] char*) | [^"u] char*) | [^"cjq] char* )? ["] space
326
+ QAPair-additional-kv ::= QAPair-additional-k ":" space value
327
+ QAPair-concise-answer-kv ::= "\"concise_answer\"" space ":" space string
328
+ QAPair-justification-kv ::= "\"justification\"" space ":" space string
329
+ QAPair-question-kv ::= "\"question\"" space ":" space string
330
+ additional-k ::= ["] ( [k] ([e] ([y] ([_] ([f] ([a] ([c] ([t] ([s] char+ | [^"s] char*) | [^"t] char*) | [^"c] char*) | [^"a] char*) | [^"f] char*) | [^"_] char*) | [^"y] char*) | [^"e] char*) | [q] ([u] ([e] ([s] ([t] ([i] ([o] ([n] ([_] ([a] ([n] ([s] ([w] ([e] ([r] ([s] char+ | [^"s] char*) | [^"r] char*) | [^"e] char*) | [^"w] char*) | [^"s] char*) | [^"n] char*) | [^"a] char*) | [^"_] char*) | [^"n] char*) | [^"o] char*) | [^"i] char*) | [^"t] char*) | [^"s] char*) | [^"e] char*) | [^"u] char*) | [^"kq] char* )? ["] space
331
+ additional-kv ::= additional-k ":" space value
332
+ array ::= "[" space ( value ("," space value)* )? "]" space
333
+ boolean ::= ("true" | "false") space
334
+ char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
335
+ decimal-part ::= [0-9]{1,16}
336
+ dot ::= [^\x0A\x0D]
337
+ integral-part ::= [0] | [1-9] [0-9]{0,15}
338
+ key-facts ::= "[" space (key-facts-item ("," space key-facts-item)*)? "]" space
339
+ key-facts-item ::= "\"" "- " key-facts-item-1{5,} "\"" space
340
+ key-facts-item-1 ::= dot
341
+ key-facts-kv ::= "\"key_facts\"" space ":" space key-facts
342
+ null ::= "null" space
343
+ number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
344
+ object ::= "{" space ( string ":" space value ("," space string ":" space value)* )? "}" space
345
+ question-answers ::= "[" space (question-answers-item ("," space question-answers-item)*)? "]" space
346
+ question-answers-item ::= "[" space question-answers-item-item ("," space question-answers-item-item){4,} "]" space
347
+ question-answers-item-item ::= QAPair
348
+ question-answers-kv ::= "\"question_answers\"" space ":" space question-answers
349
+ root ::= "{" space key-facts-kv "," space question-answers-kv ( "," space ( additional-kv ( "," space additional-kv )* ) )? "}" space
350
+ space ::= | " " | "\n" [ \t]{0,20}
351
+ string ::= "\"" char* "\"" space
352
+ value ::= object | array | string | number | boolean | null
353
+ ```
354
+
355
+ </details>
356
+
357
+ If you're using [Zod](https://zod.dev/), you can make your objects to explicitly allow extra properties w/ `nonstrict()` / `passthrough()` (or explicitly no extra props w/ `z.object(...).strict()` or `z.strictObject(...)`) but note that [zod-to-json-schema](https://github.com/StefanTerdell/zod-to-json-schema) currently always sets `"additionalProperties": false` anyway.
358
+
359
+ ```js
360
+ import { z } from 'zod';
361
+ import { zodToJsonSchema } from 'zod-to-json-schema';
362
+
363
+ const Foo = z.object({
364
+ age: z.number().positive(),
365
+ email: z.string().email(),
366
+ }).strict();
367
+
368
+ console.log(zodToJsonSchema(Foo));
369
+ ```
370
+
371
+ <details>
372
+ <summary>Show JSON schema & grammar</summary>
373
+
374
+ ```json
375
+ {
376
+ "type": "object",
377
+ "properties": {
378
+ "age": {
379
+ "type": "number",
380
+ "exclusiveMinimum": 0
381
+ },
382
+ "email": {
383
+ "type": "string",
384
+ "format": "email"
385
+ }
386
+ },
387
+ "required": [
388
+ "age",
389
+ "email"
390
+ ],
391
+ "additionalProperties": false,
392
+ "$schema": "http://json-schema.org/draft-07/schema#"
393
+ }
394
+ ```
395
+
396
+ ```
397
+ age-kv ::= "\"age\"" space ":" space number
398
+ char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
399
+ decimal-part ::= [0-9]{1,16}
400
+ email-kv ::= "\"email\"" space ":" space string
401
+ integral-part ::= [0] | [1-9] [0-9]{0,15}
402
+ number ::= ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space
403
+ root ::= "{" space age-kv "," space email-kv "}" space
404
+ space ::= | " " | "\n" [ \t]{0,20}
405
+ string ::= "\"" char* "\"" space
406
+ ```
407
+
408
+ </details>
grammars/arithmetic.gbnf ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ root ::= (expr "=" ws term "\n")+
2
+ expr ::= term ([-+*/] term)*
3
+ term ::= ident | num | "(" ws expr ")" ws
4
+ ident ::= [a-z] [a-z0-9_]* ws
5
+ num ::= [0-9]+ ws
6
+ ws ::= [ \t\n]*
grammars/c.gbnf ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ root ::= (declaration)*
2
+
3
+ declaration ::= dataType identifier "(" parameter? ")" "{" statement* "}"
4
+
5
+ dataType ::= "int" ws | "float" ws | "char" ws
6
+ identifier ::= [a-zA-Z_] [a-zA-Z_0-9]*
7
+
8
+ parameter ::= dataType identifier
9
+
10
+ statement ::=
11
+ ( dataType identifier ws "=" ws expression ";" ) |
12
+ ( identifier ws "=" ws expression ";" ) |
13
+ ( identifier ws "(" argList? ")" ";" ) |
14
+ ( "return" ws expression ";" ) |
15
+ ( "while" "(" condition ")" "{" statement* "}" ) |
16
+ ( "for" "(" forInit ";" ws condition ";" ws forUpdate ")" "{" statement* "}" ) |
17
+ ( "if" "(" condition ")" "{" statement* "}" ("else" "{" statement* "}")? ) |
18
+ ( singleLineComment ) |
19
+ ( multiLineComment )
20
+
21
+ forInit ::= dataType identifier ws "=" ws expression | identifier ws "=" ws expression
22
+ forUpdate ::= identifier ws "=" ws expression
23
+
24
+ condition ::= expression relationOperator expression
25
+ relationOperator ::= ("<=" | "<" | "==" | "!=" | ">=" | ">")
26
+
27
+ expression ::= term (("+" | "-") term)*
28
+ term ::= factor(("*" | "/") factor)*
29
+
30
+ factor ::= identifier | number | unaryTerm | funcCall | parenExpression
31
+ unaryTerm ::= "-" factor
32
+ funcCall ::= identifier "(" argList? ")"
33
+ parenExpression ::= "(" ws expression ws ")"
34
+
35
+ argList ::= expression ("," ws expression)*
36
+
37
+ number ::= [0-9]+
38
+
39
+ singleLineComment ::= "//" [^\n]* "\n"
40
+ multiLineComment ::= "/*" ( [^*] | ("*" [^/]) )* "*/"
41
+
42
+ ws ::= ([ \t\n]+)
grammars/chess.gbnf ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Specifies chess moves as a list in algebraic notation, using PGN conventions
2
+
3
+ # Force first move to "1. ", then any 1-2 digit number after, relying on model to follow the pattern
4
+ root ::= "1. " move " " move "\n" ([1-9] [0-9]? ". " move " " move "\n")+
5
+ move ::= (pawn | nonpawn | castle) [+#]?
6
+
7
+ # piece type, optional file/rank, optional capture, dest file & rank
8
+ nonpawn ::= [NBKQR] [a-h]? [1-8]? "x"? [a-h] [1-8]
9
+
10
+ # optional file & capture, dest file & rank, optional promotion
11
+ pawn ::= ([a-h] "x")? [a-h] [1-8] ("=" [NBKQR])?
12
+
13
+ castle ::= "O-O" "-O"?
grammars/english.gbnf ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ # note: this might be incomplete, mostly an example
2
+ root ::= en-char+ ([ \t\n] en-char+)*
3
+ en-char ::= letter | digit | punctuation
4
+ letter ::= [a-zA-Z]
5
+ digit ::= [0-9]
6
+ punctuation ::= [!"#$%&'()*+,-./:;<=>?@[\\\]^_`{|}~]
grammars/japanese.gbnf ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ # A probably incorrect grammar for Japanese
2
+ root ::= jp-char+ ([ \t\n] jp-char+)*
3
+ jp-char ::= hiragana | katakana | punctuation | cjk
4
+ hiragana ::= [ぁ-ゟ]
5
+ katakana ::= [ァ-ヿ]
6
+ punctuation ::= [、-〾]
7
+ cjk ::= [一-鿿]
grammars/json.gbnf ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ root ::= object
2
+ value ::= object | array | string | number | ("true" | "false" | "null") ws
3
+
4
+ object ::=
5
+ "{" ws (
6
+ string ":" ws value
7
+ ("," ws string ":" ws value)*
8
+ )? "}" ws
9
+
10
+ array ::=
11
+ "[" ws (
12
+ value
13
+ ("," ws value)*
14
+ )? "]" ws
15
+
16
+ string ::=
17
+ "\"" (
18
+ [^"\\\x7F\x00-\x1F] |
19
+ "\\" (["\\bfnrt] | "u" [0-9a-fA-F]{4}) # escapes
20
+ )* "\"" ws
21
+
22
+ number ::= ("-"? ([0-9] | [1-9] [0-9]{0,15})) ("." [0-9]+)? ([eE] [-+]? [0-9] [1-9]{0,15})? ws
23
+
24
+ # Optional space: by convention, applied in this grammar after literal chars when allowed
25
+ ws ::= | " " | "\n" [ \t]{0,20}
grammars/json_arr.gbnf ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # This is the same as json.gbnf but we restrict whitespaces at the end of the root array
2
+ # Useful for generating JSON arrays
3
+
4
+ root ::= arr
5
+ value ::= object | array | string | number | ("true" | "false" | "null") ws
6
+
7
+ arr ::=
8
+ "[\n" ws (
9
+ value
10
+ (",\n" ws value)*
11
+ )? "]"
12
+
13
+ object ::=
14
+ "{" ws (
15
+ string ":" ws value
16
+ ("," ws string ":" ws value)*
17
+ )? "}" ws
18
+
19
+ array ::=
20
+ "[" ws (
21
+ value
22
+ ("," ws value)*
23
+ )? "]" ws
24
+
25
+ string ::=
26
+ "\"" (
27
+ [^"\\\x7F\x00-\x1F] |
28
+ "\\" (["\\bfnrt] | "u" [0-9a-fA-F]{4}) # escapes
29
+ )* "\"" ws
30
+
31
+ number ::= ("-"? ([0-9] | [1-9] [0-9]{0,15})) ("." [0-9]+)? ([eE] [-+]? [1-9] [0-9]{0,15})? ws
32
+
33
+ # Optional space: by convention, applied in this grammar after literal chars when allowed
34
+ ws ::= | " " | "\n" [ \t]{0,20}
grammars/list.gbnf ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ root ::= item+
2
+
3
+ # Excludes various line break characters
4
+ item ::= "- " [^\r\n\x0b\x0c\x85\u2028\u2029]+ "\n"
include/llama-cpp.h ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma once
2
+
3
+ #ifndef __cplusplus
4
+ #error "This header is for C++ only"
5
+ #endif
6
+
7
+ #include <memory>
8
+
9
+ #include "llama.h"
10
+
11
+ struct llama_model_deleter {
12
+ void operator()(llama_model * model) { llama_model_free(model); }
13
+ };
14
+
15
+ struct llama_context_deleter {
16
+ void operator()(llama_context * context) { llama_free(context); }
17
+ };
18
+
19
+ struct llama_sampler_deleter {
20
+ void operator()(llama_sampler * sampler) { llama_sampler_free(sampler); }
21
+ };
22
+
23
+ struct llama_adapter_lora_deleter {
24
+ void operator()(llama_adapter_lora * adapter) { llama_adapter_lora_free(adapter); }
25
+ };
26
+
27
+ typedef std::unique_ptr<llama_model, llama_model_deleter> llama_model_ptr;
28
+ typedef std::unique_ptr<llama_context, llama_context_deleter> llama_context_ptr;
29
+ typedef std::unique_ptr<llama_sampler, llama_sampler_deleter> llama_sampler_ptr;
30
+ typedef std::unique_ptr<llama_adapter_lora, llama_adapter_lora_deleter> llama_adapter_lora_ptr;
include/llama.h ADDED
@@ -0,0 +1,1638 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #ifndef LLAMA_H
2
+ #define LLAMA_H
3
+
4
+ #include "ggml.h"
5
+ #include "ggml-cpu.h"
6
+ #include "ggml-backend.h"
7
+ #include "ggml-opt.h"
8
+ #include "gguf.h"
9
+
10
+ #include <stddef.h>
11
+ #include <stdint.h>
12
+ #include <stdio.h>
13
+ #include <stdbool.h>
14
+
15
+ #ifdef LLAMA_SHARED
16
+ # if defined(_WIN32) && !defined(__MINGW32__)
17
+ # ifdef LLAMA_BUILD
18
+ # define LLAMA_API __declspec(dllexport)
19
+ # else
20
+ # define LLAMA_API __declspec(dllimport)
21
+ # endif
22
+ # else
23
+ # define LLAMA_API __attribute__ ((visibility ("default")))
24
+ # endif
25
+ #else
26
+ # define LLAMA_API
27
+ #endif
28
+
29
+ #ifdef __GNUC__
30
+ # define DEPRECATED(func, hint) func __attribute__((deprecated(hint)))
31
+ #elif defined(_MSC_VER)
32
+ # define DEPRECATED(func, hint) __declspec(deprecated(hint)) func
33
+ #else
34
+ # define DEPRECATED(func, hint) func
35
+ #endif
36
+
37
+ #define LLAMA_DEFAULT_SEED 0xFFFFFFFF
38
+
39
+ #define LLAMA_TOKEN_NULL -1
40
+
41
+ #define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'
42
+ #define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'
43
+ #define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'
44
+
45
+ #define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN
46
+ #define LLAMA_SESSION_VERSION 10
47
+
48
+ #define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ
49
+ #define LLAMA_STATE_SEQ_VERSION 3
50
+
51
+ #ifdef __cplusplus
52
+ extern "C" {
53
+ #endif
54
+
55
+ //
56
+ // C interface
57
+ //
58
+ // TODO: show sample usage
59
+ //
60
+
61
+ struct llama_vocab;
62
+ struct llama_model;
63
+ struct llama_context;
64
+ struct llama_sampler;
65
+
66
+ typedef struct llama_memory_i * llama_memory_t;
67
+
68
+ typedef int32_t llama_pos;
69
+ typedef int32_t llama_token;
70
+ typedef int32_t llama_seq_id;
71
+
72
+ enum llama_vocab_type {
73
+ LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab
74
+ LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback
75
+ LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE
76
+ LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece
77
+ LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram
78
+ LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization
79
+ LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming
80
+ };
81
+
82
+ enum llama_rope_type {
83
+ LLAMA_ROPE_TYPE_NONE = -1,
84
+ LLAMA_ROPE_TYPE_NORM = 0,
85
+ LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,
86
+ LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE,
87
+ LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE,
88
+ LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION,
89
+ };
90
+
91
+ enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file
92
+ LLAMA_TOKEN_TYPE_UNDEFINED = 0,
93
+ LLAMA_TOKEN_TYPE_NORMAL = 1,
94
+ LLAMA_TOKEN_TYPE_UNKNOWN = 2,
95
+ LLAMA_TOKEN_TYPE_CONTROL = 3,
96
+ LLAMA_TOKEN_TYPE_USER_DEFINED = 4,
97
+ LLAMA_TOKEN_TYPE_UNUSED = 5,
98
+ LLAMA_TOKEN_TYPE_BYTE = 6,
99
+ };
100
+
101
+ enum llama_token_attr {
102
+ LLAMA_TOKEN_ATTR_UNDEFINED = 0,
103
+ LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0,
104
+ LLAMA_TOKEN_ATTR_UNUSED = 1 << 1,
105
+ LLAMA_TOKEN_ATTR_NORMAL = 1 << 2,
106
+ LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL?
107
+ LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,
108
+ LLAMA_TOKEN_ATTR_BYTE = 1 << 5,
109
+ LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6,
110
+ LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7,
111
+ LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8,
112
+ LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9,
113
+ };
114
+
115
+ // model file types
116
+ enum llama_ftype {
117
+ LLAMA_FTYPE_ALL_F32 = 0,
118
+ LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors
119
+ LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors
120
+ LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors
121
+ // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
122
+ // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed
123
+ // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed
124
+ LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors
125
+ LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors
126
+ LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors
127
+ LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
128
+ LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors
129
+ LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors
130
+ LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors
131
+ LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors
132
+ LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors
133
+ LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors
134
+ LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors
135
+ LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors
136
+ LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors
137
+ LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors
138
+ LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors
139
+ LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors
140
+ LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors
141
+ LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors
142
+ LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors
143
+ LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors
144
+ LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors
145
+ LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors
146
+ LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors
147
+ LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors
148
+ LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors
149
+ LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors
150
+ //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack
151
+ //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack
152
+ //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack
153
+ LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors
154
+ LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors
155
+ LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors
156
+ LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors
157
+ LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors
158
+ LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors
159
+
160
+ LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
161
+ };
162
+
163
+ // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"
164
+ LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype);
165
+
166
+ enum llama_rope_scaling_type {
167
+ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,
168
+ LLAMA_ROPE_SCALING_TYPE_NONE = 0,
169
+ LLAMA_ROPE_SCALING_TYPE_LINEAR = 1,
170
+ LLAMA_ROPE_SCALING_TYPE_YARN = 2,
171
+ LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3,
172
+ LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE,
173
+ };
174
+
175
+ enum llama_pooling_type {
176
+ LLAMA_POOLING_TYPE_UNSPECIFIED = -1,
177
+ LLAMA_POOLING_TYPE_NONE = 0,
178
+ LLAMA_POOLING_TYPE_MEAN = 1,
179
+ LLAMA_POOLING_TYPE_CLS = 2,
180
+ LLAMA_POOLING_TYPE_LAST = 3,
181
+ LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph
182
+ };
183
+
184
+ enum llama_attention_type {
185
+ LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1,
186
+ LLAMA_ATTENTION_TYPE_CAUSAL = 0,
187
+ LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1,
188
+ };
189
+
190
+ enum llama_flash_attn_type {
191
+ LLAMA_FLASH_ATTN_TYPE_AUTO = -1,
192
+ LLAMA_FLASH_ATTN_TYPE_DISABLED = 0,
193
+ LLAMA_FLASH_ATTN_TYPE_ENABLED = 1,
194
+ };
195
+
196
+ LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type);
197
+
198
+ enum llama_split_mode {
199
+ LLAMA_SPLIT_MODE_NONE = 0, // single GPU
200
+ LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs
201
+ LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported
202
+ LLAMA_SPLIT_MODE_TENSOR = 3,
203
+ };
204
+
205
+ enum llama_load_mode {
206
+ LLAMA_LOAD_MODE_AUTO = -1, // auto-detect based on device capabilities
207
+ LLAMA_LOAD_MODE_NONE = 0, // no special loading mode
208
+ LLAMA_LOAD_MODE_MMAP = 1, // memory map the model
209
+ LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing
210
+ LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing
211
+ LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available
212
+ };
213
+
214
+ LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);
215
+ LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);
216
+
217
+ enum llama_lazy_mode {
218
+ LLAMA_LAZY_MODE_OFF = 0, // always read the whole tensor up front
219
+ LLAMA_LAZY_MODE_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)
220
+ LLAMA_LAZY_MODE_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap)
221
+ };
222
+
223
+ enum llama_context_type {
224
+ LLAMA_CONTEXT_TYPE_DEFAULT = 0,
225
+ LLAMA_CONTEXT_TYPE_MTP = 1,
226
+ };
227
+
228
+ // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979)
229
+ typedef struct llama_token_data {
230
+ llama_token id; // token id
231
+ float logit; // log-odds of the token
232
+ float p; // probability of the token
233
+ } llama_token_data;
234
+
235
+ typedef struct llama_token_data_array {
236
+ // TODO: consider SoA
237
+ // NOTE: this pointer can be modified by the samplers
238
+ llama_token_data * data;
239
+ size_t size;
240
+ int64_t selected; // this is the index in the data array (i.e. not the token id)
241
+ bool sorted; // note: do not assume the data is sorted - always check this flag
242
+ } llama_token_data_array;
243
+
244
+ typedef bool (*llama_progress_callback)(float progress, void * user_data);
245
+
246
+ // Input data for llama_encode/llama_decode
247
+ // A llama_batch object can contain input about one or many sequences
248
+ // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens
249
+ //
250
+ // - token : the token ids of the input (used when embd is NULL)
251
+ // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)
252
+ // - pos : the positions of the respective token in the sequence
253
+ // (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode)
254
+ // - seq_id : the sequence to which the respective token belongs
255
+ // (if set to NULL, the sequence ID will be assumed to be 0)
256
+ // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output
257
+ // (if set to NULL:
258
+ // - if embeddings: all tokens are output
259
+ // - if not: only the last token is output
260
+ // )
261
+ //
262
+ typedef struct llama_batch {
263
+ int32_t n_tokens;
264
+
265
+ llama_token * token;
266
+ float * embd;
267
+ llama_pos * pos;
268
+ int32_t * n_seq_id;
269
+ llama_seq_id ** seq_id;
270
+ int8_t * logits; // TODO: rename this to "output"
271
+ } llama_batch;
272
+
273
+ enum llama_model_kv_override_type {
274
+ LLAMA_KV_OVERRIDE_TYPE_INT,
275
+ LLAMA_KV_OVERRIDE_TYPE_FLOAT,
276
+ LLAMA_KV_OVERRIDE_TYPE_BOOL,
277
+ LLAMA_KV_OVERRIDE_TYPE_STR,
278
+ };
279
+
280
+ enum llama_model_meta_key {
281
+ LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE,
282
+ LLAMA_MODEL_META_KEY_SAMPLING_TOP_K,
283
+ LLAMA_MODEL_META_KEY_SAMPLING_TOP_P,
284
+ LLAMA_MODEL_META_KEY_SAMPLING_MIN_P,
285
+ LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY,
286
+ LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD,
287
+ LLAMA_MODEL_META_KEY_SAMPLING_TEMP,
288
+ LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N,
289
+ LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT,
290
+ LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT,
291
+ LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU,
292
+ LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA,
293
+ };
294
+
295
+ struct llama_model_kv_override {
296
+ enum llama_model_kv_override_type tag;
297
+
298
+ char key[128];
299
+
300
+ union {
301
+ int64_t val_i64;
302
+ double val_f64;
303
+ bool val_bool;
304
+ char val_str[128];
305
+ };
306
+ };
307
+
308
+ struct llama_model_tensor_buft_override {
309
+ const char * pattern;
310
+ ggml_backend_buffer_type_t buft;
311
+ };
312
+
313
+ struct llama_model_params {
314
+ // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)
315
+ ggml_backend_dev_t * devices;
316
+
317
+ // NULL-terminated list of buffer types to use for tensors that match a pattern
318
+ const struct llama_model_tensor_buft_override * tensor_buft_overrides;
319
+
320
+ int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers
321
+ enum llama_split_mode split_mode; // how to split the model across multiple GPUs
322
+ enum llama_load_mode load_mode; // how to load the model
323
+
324
+ enum llama_lazy_mode lazy_mode; // on-demand reading of tensors marked by the arch
325
+
326
+ // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE
327
+ int32_t main_gpu;
328
+
329
+ // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()
330
+ const float * tensor_split;
331
+
332
+ // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.
333
+ // If the provided progress_callback returns true, model loading continues.
334
+ // If it returns false, model loading is immediately aborted.
335
+ llama_progress_callback progress_callback;
336
+
337
+ // context pointer passed to the progress callback
338
+ void * progress_callback_user_data;
339
+
340
+ // override key-value pairs of the model meta data
341
+ const struct llama_model_kv_override * kv_overrides;
342
+
343
+ // Keep the booleans together to avoid misalignment during copy-by-value.
344
+ bool vocab_only; // only load the vocabulary, no weights
345
+ bool check_tensors; // validate model tensor data
346
+ bool use_extra_bufts; // use extra buffer types (used for weight repacking)
347
+ bool no_host; // bypass host buffer allowing extra buffers to be used
348
+ bool no_alloc; // only load metadata and simulate memory allocations
349
+ bool load_mtp; // whether to load MTP layers
350
+ };
351
+
352
+ struct llama_sampler_seq_config {
353
+ llama_seq_id seq_id;
354
+ struct llama_sampler * sampler;
355
+ };
356
+
357
+ // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations
358
+ // https://github.com/ggml-org/llama.cpp/pull/7544
359
+ struct llama_context_params {
360
+ uint32_t n_ctx; // text context, 0 = from model
361
+ uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode
362
+ uint32_t n_ubatch; // physical maximum batch size
363
+ uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models)
364
+ uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]
365
+ uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch)
366
+ uint32_t n_outputs_max_per_seq; // max outputs per sequence (0 = n_outputs_max)
367
+ int32_t n_threads; // number of threads to use for generation
368
+ int32_t n_threads_batch; // number of threads to use for batch processing
369
+
370
+ enum llama_context_type ctx_type; // set the context type (e.g. MTP)
371
+ enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`
372
+ enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id
373
+ enum llama_attention_type attention_type; // attention type to use for embeddings
374
+ enum llama_flash_attn_type flash_attn_type; // when to enable Flash Attention
375
+
376
+ // ref: https://github.com/ggml-org/llama.cpp/pull/2054
377
+ float rope_freq_base; // RoPE base frequency, 0 = from model
378
+ float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model
379
+ float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model
380
+ float yarn_attn_factor; // YaRN magnitude scaling factor
381
+ float yarn_beta_fast; // YaRN low correction dim
382
+ float yarn_beta_slow; // YaRN high correction dim
383
+ uint32_t yarn_orig_ctx; // YaRN original context size
384
+ float defrag_thold; // [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default)
385
+
386
+ ggml_backend_sched_eval_callback cb_eval;
387
+ void * cb_eval_user_data;
388
+
389
+ enum ggml_type type_k; // data type for K cache [EXPERIMENTAL]
390
+ enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]
391
+
392
+ // Abort callback
393
+ // if it returns true, execution of llama_decode() will be aborted
394
+ // currently works only with CPU execution
395
+ ggml_abort_callback abort_callback;
396
+ void * abort_callback_data;
397
+
398
+ // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value.
399
+ bool embeddings; // if true, extract embeddings (together with logits)
400
+ bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU
401
+ bool no_perf; // measure performance timings
402
+ bool op_offload; // offload host tensor operations to device
403
+ bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)
404
+ // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases
405
+ // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573
406
+ bool kv_unified; // use a unified buffer across the input sequences when computing the attention
407
+ // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix
408
+ // ref: https://github.com/ggml-org/llama.cpp/pull/14363
409
+
410
+ // [EXPERIMENTAL]
411
+ // backend sampler chain configuration (make sure the caller keeps the sampler chains alive)
412
+ // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init)
413
+ struct llama_sampler_seq_config * samplers;
414
+ size_t n_samplers;
415
+
416
+ // a source/target/parent context
417
+ // can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts
418
+ struct llama_context * ctx_other;
419
+ };
420
+
421
+ struct llama_model_tensor_override {
422
+ const char * pattern;
423
+ enum ggml_type type;
424
+ };
425
+
426
+ struct llama_model_imatrix_data {
427
+ const char * name;
428
+ const float * data;
429
+ size_t size;
430
+ };
431
+
432
+ // model quantization parameters
433
+ typedef struct llama_model_quantize_params {
434
+ int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
435
+ enum llama_ftype ftype; // quantize to this llama_ftype
436
+ enum ggml_type output_tensor_type; // output tensor type
437
+ enum ggml_type token_embedding_type; // token embeddings tensor type
438
+ bool allow_requantize; // allow quantizing non-f32/f16 tensors
439
+ bool quantize_output_tensor; // quantize output.weight
440
+ bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
441
+ bool pure; // quantize all tensors to the default type
442
+ bool keep_split; // quantize to the same number of shards
443
+ bool dry_run; // calculate and show the final quantization size without performing quantization
444
+ const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data
445
+ const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides
446
+ const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides
447
+ const int32_t * prune_layers; // pointer to layer indices to prune
448
+ size_t max_buf_size; // max bytes of tensor rows kept in memory at once, 0 = default (8 GiB)
449
+ } llama_model_quantize_params;
450
+
451
+ typedef struct llama_logit_bias {
452
+ llama_token token;
453
+ float bias;
454
+ } llama_logit_bias;
455
+
456
+ typedef struct llama_sampler_chain_params {
457
+ bool no_perf; // whether to measure performance timings
458
+ } llama_sampler_chain_params;
459
+
460
+ // used in chat template
461
+ typedef struct llama_chat_message {
462
+ const char * role;
463
+ const char * content;
464
+ } llama_chat_message;
465
+
466
+ // lora adapter
467
+ struct llama_adapter_lora;
468
+
469
+ LLAMA_API const char * llama_version(void);
470
+
471
+ // Helpers for getting default parameters
472
+ // TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172)
473
+ LLAMA_API struct llama_model_params llama_model_default_params(void);
474
+ LLAMA_API struct llama_context_params llama_context_default_params(void);
475
+ LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void);
476
+ LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);
477
+
478
+ // Initialize the llama + ggml backend
479
+ // If numa is true, use NUMA optimizations
480
+ // Call once at the start of the program
481
+ LLAMA_API void llama_backend_init(void);
482
+
483
+ // Call once at the end of the program - currently only used for MPI
484
+ LLAMA_API void llama_backend_free(void);
485
+
486
+ //optional:
487
+ LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa);
488
+
489
+ // Optional: an auto threadpool gets created in ggml if not passed explicitly
490
+ LLAMA_API void llama_attach_threadpool(
491
+ struct llama_context * ctx,
492
+ ggml_threadpool_t threadpool,
493
+ ggml_threadpool_t threadpool_batch);
494
+
495
+ LLAMA_API void llama_detach_threadpool(struct llama_context * ctx);
496
+
497
+ typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata);
498
+
499
+ // Create a new model from GGUF metadata as well as a function to set the tensor data
500
+ // - tensors are created as GGML_TYPE_F32 by default,
501
+ // override by adding a tensor with the same name but a different name to the context
502
+ LLAMA_API struct llama_model * llama_model_init_from_user(
503
+ struct gguf_context * metadata,
504
+ llama_model_set_tensor_data_t set_tensor_data, // function to initialize tensor data with
505
+ void * set_tensor_data_ud, // userdata for function
506
+ struct llama_model_params params);
507
+
508
+ DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file(
509
+ const char * path_model,
510
+ struct llama_model_params params),
511
+ "use llama_model_load_from_file instead");
512
+
513
+ // Load a model from a file
514
+ // If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf
515
+ // If the split file name does not follow this pattern, use llama_model_load_from_splits
516
+ LLAMA_API struct llama_model * llama_model_load_from_file(
517
+ const char * path_model,
518
+ struct llama_model_params params);
519
+
520
+ // Load a model from an open FILE pointer
521
+ LLAMA_API struct llama_model * llama_model_load_from_file_ptr(
522
+ FILE * file,
523
+ struct llama_model_params params);
524
+
525
+ // Load a model from multiple splits (support custom naming scheme)
526
+ // The paths must be in the correct order
527
+ LLAMA_API struct llama_model * llama_model_load_from_splits(
528
+ const char ** paths,
529
+ size_t n_paths,
530
+ struct llama_model_params params);
531
+
532
+ LLAMA_API void llama_model_save_to_file(
533
+ const struct llama_model * model,
534
+ const char * path_model);
535
+
536
+ DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model),
537
+ "use llama_model_free instead");
538
+
539
+ LLAMA_API void llama_model_free(struct llama_model * model);
540
+
541
+ LLAMA_API struct llama_context * llama_init_from_model(
542
+ struct llama_model * model,
543
+ struct llama_context_params params);
544
+
545
+ DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model(
546
+ struct llama_model * model,
547
+ struct llama_context_params params),
548
+ "use llama_init_from_model instead");
549
+
550
+ // Frees all allocated memory
551
+ LLAMA_API void llama_free(struct llama_context * ctx);
552
+
553
+ LLAMA_API int64_t llama_time_us(void);
554
+
555
+ LLAMA_API size_t llama_max_devices(void);
556
+ LLAMA_API size_t llama_max_parallel_sequences(void);
557
+ LLAMA_API size_t llama_max_tensor_buft_overrides(void);
558
+
559
+ LLAMA_API bool llama_supports_mmap (void);
560
+ LLAMA_API bool llama_supports_mlock (void);
561
+ LLAMA_API bool llama_supports_gpu_offload(void);
562
+ LLAMA_API bool llama_supports_rpc (void);
563
+
564
+ // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions
565
+ // In some cases the requested values via llama_context_params may differ from the actual values used by the context
566
+ // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732
567
+ LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx);
568
+ LLAMA_API uint32_t llama_n_ctx_seq (const struct llama_context * ctx);
569
+ LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);
570
+ LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx);
571
+ LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx);
572
+ LLAMA_API uint32_t llama_n_rs_seq (const struct llama_context * ctx);
573
+
574
+ DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead");
575
+ DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead");
576
+ DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead");
577
+ DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead");
578
+
579
+ DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead");
580
+
581
+ LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx);
582
+ LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx);
583
+ LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type
584
+
585
+ LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model);
586
+ LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model);
587
+
588
+ LLAMA_API int32_t llama_model_n_ctx_train (const struct llama_model * model);
589
+ LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model);
590
+ LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model);
591
+ LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model);
592
+ LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model);
593
+ LLAMA_API int32_t llama_model_n_layer_nextn(const struct llama_model * model);
594
+ LLAMA_API int32_t llama_model_n_head (const struct llama_model * model);
595
+ LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model);
596
+ LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model);
597
+
598
+ // Get the model's RoPE frequency scaling factor
599
+ LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model);
600
+
601
+ // Returns the number of classifier outputs (only valid for classifier models)
602
+ // Undefined behavior for non-classifier models
603
+ LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model);
604
+
605
+ // Returns label of classifier output by index (<n_cls_out). Returns nullptr if no label provided
606
+ LLAMA_API const char * llama_model_cls_label(const struct llama_model * model, uint32_t i);
607
+
608
+ LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_vocab * vocab);
609
+
610
+ LLAMA_API int32_t llama_vocab_n_tokens(const struct llama_vocab * vocab);
611
+
612
+ // Functions to access the model's GGUF metadata scalar values
613
+ // - The functions return the length of the string on success, or -1 on failure
614
+ // - The output string is always null-terminated and cleared on failure
615
+ // - When retrieving a string, an extra byte must be allocated to account for the null terminator
616
+ // - GGUF array values are not supported by these functions
617
+
618
+ // Get metadata value as a string by key name
619
+ LLAMA_API int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);
620
+
621
+ // Get the number of metadata key/value pairs
622
+ LLAMA_API int32_t llama_model_meta_count(const struct llama_model * model);
623
+
624
+ // Get sampling metadata key name. Returns nullptr if the key is invalid
625
+ LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key);
626
+
627
+ // Get metadata key name by index
628
+ LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
629
+
630
+ // Get metadata value as a string by index
631
+ LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);
632
+
633
+ // Get a string describing the model type
634
+ LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);
635
+
636
+ // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0
637
+ LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model);
638
+
639
+ // Returns the total size of all the tensors in the model in bytes
640
+ LLAMA_API uint64_t llama_model_size(const struct llama_model * model);
641
+
642
+ // Get the default chat template. Returns nullptr if not available
643
+ // If name is NULL, returns the default chat template
644
+ LLAMA_API const char * llama_model_chat_template(const struct llama_model * model, const char * name);
645
+
646
+ // Returns the total number of parameters in the model
647
+ LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model);
648
+
649
+ // Returns true if the model contains an encoder that requires llama_encode() call
650
+ LLAMA_API bool llama_model_has_encoder(const struct llama_model * model);
651
+
652
+ // Returns true if the model contains a decoder that requires llama_decode() call
653
+ LLAMA_API bool llama_model_has_decoder(const struct llama_model * model);
654
+
655
+ // For encoder-decoder models, this function returns id of the token that must be provided
656
+ // to the decoder to start generating output sequence. For other models, it returns -1.
657
+ LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model);
658
+
659
+ // Returns true if the model is recurrent (like Mamba, RWKV, etc.)
660
+ LLAMA_API bool llama_model_is_recurrent(const struct llama_model * model);
661
+
662
+ // Returns true if the model is hybrid (like Jamba, Granite, etc.)
663
+ LLAMA_API bool llama_model_is_hybrid(const struct llama_model * model);
664
+
665
+ // Returns true if the model is diffusion-based (like LLaDA, Dream, etc.)
666
+ LLAMA_API bool llama_model_is_diffusion(const struct llama_model * model);
667
+
668
+ // Returns 0 on success
669
+ LLAMA_API uint32_t llama_model_quantize(
670
+ const char * fname_inp,
671
+ const char * fname_out,
672
+ const llama_model_quantize_params * params);
673
+
674
+ //
675
+ // Adapters
676
+ //
677
+
678
+ // Load a LoRA adapter from file
679
+ // The adapter is valid as long as the associated model is not freed
680
+ LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init(
681
+ struct llama_model * model,
682
+ const char * path_lora);
683
+
684
+ // Functions to access the adapter's GGUF metadata scalar values
685
+ // - The functions return the length of the string on success, or -1 on failure
686
+ // - The output string is always null-terminated and cleared on failure
687
+ // - When retrieving a string, an extra byte must be allocated to account for the null terminator
688
+ // - GGUF array values are not supported by these functions
689
+
690
+ // Get metadata value as a string by key name
691
+ LLAMA_API int32_t llama_adapter_meta_val_str(const struct llama_adapter_lora * adapter, const char * key, char * buf, size_t buf_size);
692
+
693
+ // Get the number of metadata key/value pairs
694
+ LLAMA_API int32_t llama_adapter_meta_count(const struct llama_adapter_lora * adapter);
695
+
696
+ // Get metadata key name by index
697
+ LLAMA_API int32_t llama_adapter_meta_key_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);
698
+
699
+ // Get metadata value as a string by index
700
+ LLAMA_API int32_t llama_adapter_meta_val_str_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);
701
+
702
+ // Manually free a LoRA adapter
703
+ // NOTE: loaded adapters that are not manually freed will be freed when the associated model is deleted
704
+ LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter);
705
+
706
+ // Get the invocation tokens if the current lora is an alora
707
+ LLAMA_API uint64_t llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter);
708
+ LLAMA_API const llama_token * llama_adapter_get_alora_invocation_tokens (const struct llama_adapter_lora * adapter);
709
+
710
+ // The following functions operate on a llama_context, hence the naming: llama_verb_...
711
+
712
+ // Set LoRa adapters on the context. Will only modify if the adapters currently in context are different.
713
+ LLAMA_API int32_t llama_set_adapters_lora(
714
+ struct llama_context * ctx,
715
+ struct llama_adapter_lora ** adapters,
716
+ size_t n_adapters,
717
+ float * scales);
718
+
719
+ // Apply a loaded control vector to a llama_context, or if data is NULL, clear
720
+ // the currently loaded vector.
721
+ // n_embd should be the size of a single layer's control, and data should point
722
+ // to an n_embd x n_layers buffer starting from layer 1.
723
+ // il_start and il_end are the layer range the vector should apply to (both inclusive)
724
+ // See llama_control_vector_load in common to load a control vector.
725
+ LLAMA_API int32_t llama_set_adapter_cvec(
726
+ struct llama_context * ctx,
727
+ const float * data,
728
+ size_t len,
729
+ int32_t n_embd,
730
+ int32_t il_start,
731
+ int32_t il_end);
732
+
733
+ //
734
+ // Memory
735
+ //
736
+
737
+ // Clear the memory contents
738
+ // If data == true, the data buffers will also be cleared together with the metadata
739
+ LLAMA_API void llama_memory_clear(
740
+ llama_memory_t mem,
741
+ bool data);
742
+
743
+ // Removes all tokens that belong to the specified sequence and have positions in [p0, p1)
744
+ // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails
745
+ // seq_id < 0 : match any sequence [TAG_LLAMA_SEQ_ID_NEG]
746
+ // p0 < 0 : [0, p1]
747
+ // p1 < 0 : [p0, inf)
748
+ LLAMA_API bool llama_memory_seq_rm(
749
+ llama_memory_t mem,
750
+ llama_seq_id seq_id,
751
+ llama_pos p0,
752
+ llama_pos p1);
753
+
754
+ // Copy all tokens that belong to the specified sequence to another sequence
755
+ // p0 < 0 : [0, p1]
756
+ // p1 < 0 : [p0, inf)
757
+ LLAMA_API void llama_memory_seq_cp(
758
+ llama_memory_t mem,
759
+ llama_seq_id seq_id_src,
760
+ llama_seq_id seq_id_dst,
761
+ llama_pos p0,
762
+ llama_pos p1);
763
+
764
+ // Removes all tokens that do not belong to the specified sequence
765
+ LLAMA_API void llama_memory_seq_keep(
766
+ llama_memory_t mem,
767
+ llama_seq_id seq_id);
768
+
769
+ // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)
770
+ // p0 < 0 : [0, p1]
771
+ // p1 < 0 : [p0, inf)
772
+ LLAMA_API void llama_memory_seq_add(
773
+ llama_memory_t mem,
774
+ llama_seq_id seq_id,
775
+ llama_pos p0,
776
+ llama_pos p1,
777
+ llama_pos delta);
778
+
779
+ // Integer division of the positions by factor of `d > 1`
780
+ // p0 < 0 : [0, p1]
781
+ // p1 < 0 : [p0, inf)
782
+ LLAMA_API void llama_memory_seq_div(
783
+ llama_memory_t mem,
784
+ llama_seq_id seq_id,
785
+ llama_pos p0,
786
+ llama_pos p1,
787
+ int d);
788
+
789
+ // Returns the smallest position present in the memory for the specified sequence
790
+ // This is typically non-zero only for SWA caches
791
+ // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
792
+ // Return -1 if the sequence is empty
793
+ LLAMA_API llama_pos llama_memory_seq_pos_min(
794
+ llama_memory_t mem,
795
+ llama_seq_id seq_id);
796
+
797
+ // Returns the largest position present in the memory for the specified sequence
798
+ // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory
799
+ // Return -1 if the sequence is empty
800
+ LLAMA_API llama_pos llama_memory_seq_pos_max(
801
+ llama_memory_t mem,
802
+ llama_seq_id seq_id);
803
+
804
+ // Check if the memory supports shifting
805
+ LLAMA_API bool llama_memory_can_shift(llama_memory_t mem);
806
+
807
+ //
808
+ // State / sessions
809
+ //
810
+
811
+ // Returns the *actual* size in bytes of the state
812
+ // (logits, embedding and memory)
813
+ // Only use when saving the state, not when restoring it, otherwise the size may be too small.
814
+ LLAMA_API size_t llama_state_get_size(struct llama_context * ctx);
815
+ LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx),
816
+ "use llama_state_get_size instead");
817
+
818
+ // Copies the state to the specified destination address.
819
+ // Destination needs to have allocated enough memory.
820
+ // Returns the number of bytes copied
821
+ LLAMA_API size_t llama_state_get_data(
822
+ struct llama_context * ctx,
823
+ uint8_t * dst,
824
+ size_t size);
825
+ LLAMA_API DEPRECATED(size_t llama_copy_state_data(
826
+ struct llama_context * ctx,
827
+ uint8_t * dst),
828
+ "use llama_state_get_data instead");
829
+
830
+ // Set the state reading from the specified address
831
+ // Returns the number of bytes read
832
+ LLAMA_API size_t llama_state_set_data(
833
+ struct llama_context * ctx,
834
+ const uint8_t * src,
835
+ size_t size);
836
+ LLAMA_API DEPRECATED(size_t llama_set_state_data(
837
+ struct llama_context * ctx,
838
+ const uint8_t * src),
839
+ "use llama_state_set_data instead");
840
+
841
+ // Save/load session file
842
+ LLAMA_API bool llama_state_load_file(
843
+ struct llama_context * ctx,
844
+ const char * path_session,
845
+ llama_token * tokens_out,
846
+ size_t n_token_capacity,
847
+ size_t * n_token_count_out);
848
+ LLAMA_API DEPRECATED(bool llama_load_session_file(
849
+ struct llama_context * ctx,
850
+ const char * path_session,
851
+ llama_token * tokens_out,
852
+ size_t n_token_capacity,
853
+ size_t * n_token_count_out),
854
+ "use llama_state_load_file instead");
855
+
856
+ LLAMA_API bool llama_state_save_file(
857
+ struct llama_context * ctx,
858
+ const char * path_session,
859
+ const llama_token * tokens,
860
+ size_t n_token_count);
861
+ LLAMA_API DEPRECATED(bool llama_save_session_file(
862
+ struct llama_context * ctx,
863
+ const char * path_session,
864
+ const llama_token * tokens,
865
+ size_t n_token_count),
866
+ "use llama_state_save_file instead");
867
+
868
+ // Get the exact size needed to copy the state of a single sequence
869
+ LLAMA_API size_t llama_state_seq_get_size(
870
+ struct llama_context * ctx,
871
+ llama_seq_id seq_id);
872
+
873
+ // Copy the state of a single sequence into the specified buffer
874
+ LLAMA_API size_t llama_state_seq_get_data(
875
+ struct llama_context * ctx,
876
+ uint8_t * dst,
877
+ size_t size,
878
+ llama_seq_id seq_id);
879
+
880
+ // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence
881
+ // Returns:
882
+ // - Positive: Ok
883
+ // - Zero: Failed to load
884
+ LLAMA_API size_t llama_state_seq_set_data(
885
+ struct llama_context * ctx,
886
+ const uint8_t * src,
887
+ size_t size,
888
+ llama_seq_id dest_seq_id);
889
+
890
+ LLAMA_API size_t llama_state_seq_save_file(
891
+ struct llama_context * ctx,
892
+ const char * filepath,
893
+ llama_seq_id seq_id,
894
+ const llama_token * tokens,
895
+ size_t n_token_count);
896
+
897
+ // If tokens_out is NULL, only the token count is reported through n_token_count_out and no state is loaded
898
+ LLAMA_API size_t llama_state_seq_load_file(
899
+ struct llama_context * ctx,
900
+ const char * filepath,
901
+ llama_seq_id dest_seq_id,
902
+ llama_token * tokens_out,
903
+ size_t n_token_capacity,
904
+ size_t * n_token_count_out);
905
+
906
+ #define LLAMA_STATE_SEQ_FLAGS_NONE 0
907
+
908
+ // for backwards-compat
909
+ #define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1
910
+
911
+ // work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba)
912
+ #define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1
913
+
914
+ // Keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load).
915
+ // Getting the state for a seq_id with this flag invalidates all prior states gotten for that seq_id with this flag.
916
+ #define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2
917
+
918
+ typedef uint32_t llama_state_seq_flags;
919
+
920
+ LLAMA_API size_t llama_state_seq_get_size_ext(
921
+ struct llama_context * ctx,
922
+ llama_seq_id seq_id,
923
+ llama_state_seq_flags flags);
924
+
925
+ LLAMA_API size_t llama_state_seq_get_data_ext(
926
+ struct llama_context * ctx,
927
+ uint8_t * dst,
928
+ size_t size,
929
+ llama_seq_id seq_id,
930
+ llama_state_seq_flags flags);
931
+
932
+ LLAMA_API size_t llama_state_seq_set_data_ext(
933
+ struct llama_context * ctx,
934
+ const uint8_t * src,
935
+ size_t size,
936
+ llama_seq_id dest_seq_id,
937
+ llama_state_seq_flags flags);
938
+
939
+ //
940
+ // Decoding
941
+ //
942
+
943
+ // Return batch for single sequence of tokens
944
+ // The sequence ID will be fixed to 0
945
+ // The position of the tokens will be tracked automatically by llama_decode
946
+ //
947
+ // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
948
+ //
949
+ LLAMA_API struct llama_batch llama_batch_get_one(
950
+ llama_token * tokens,
951
+ int32_t n_tokens);
952
+
953
+ // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
954
+ // Each token can be assigned up to n_seq_max sequence ids
955
+ // The batch has to be freed with llama_batch_free()
956
+ // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)
957
+ // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
958
+ // The rest of the llama_batch members are allocated with size n_tokens
959
+ // All members are left uninitialized
960
+ LLAMA_API struct llama_batch llama_batch_init(
961
+ int32_t n_tokens,
962
+ int32_t embd,
963
+ int32_t n_seq_max);
964
+
965
+ // Frees a batch of tokens allocated with llama_batch_init()
966
+ LLAMA_API void llama_batch_free(struct llama_batch batch);
967
+
968
+ // Process a batch of tokens.
969
+ // In contrast to llama_decode() - this call does not use KV cache.
970
+ // For encode-decoder contexts, processes the batch using the encoder.
971
+ // Can store the encoder output internally for later use by the decoder's cross-attention layers.
972
+ // 0 - success
973
+ // < 0 - error. the memory state is restored to the state before this call
974
+ LLAMA_API int32_t llama_encode(
975
+ struct llama_context * ctx,
976
+ struct llama_batch batch);
977
+
978
+ // Process a batch of tokens.
979
+ // Requires the context to have a memory.
980
+ // For encode-decoder contexts, processes the batch using the decoder.
981
+ // Positive return values does not mean a fatal error, but rather a warning.
982
+ // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context
983
+ // To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max()
984
+ // Upon other return values, the memory state is restored to the state before this call
985
+ // 0 - success
986
+ // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)
987
+ // 2 - aborted (processed ubatches will remain in the context's memory)
988
+ // -1 - invalid input batch
989
+ // < -1 - fatal error (processed ubatches will remain in the context's memory)
990
+ LLAMA_API int32_t llama_decode(
991
+ struct llama_context * ctx,
992
+ struct llama_batch batch);
993
+
994
+ // Set the number of threads used for decoding
995
+ // n_threads is the number of threads used for generation (single token)
996
+ // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)
997
+ LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch);
998
+
999
+ // Get the number of threads used for generation of a single token.
1000
+ LLAMA_API int32_t llama_n_threads(struct llama_context * ctx);
1001
+
1002
+ // Get the number of threads used for prompt and batch processing (multiple token).
1003
+ LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx);
1004
+
1005
+ // Set whether the context outputs embeddings or not
1006
+ // TODO: rename to avoid confusion with llama_get_embeddings()
1007
+ LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings);
1008
+
1009
+ // Set whether to use causal attention or not
1010
+ // If set to true, the model will only attend to the past tokens
1011
+ LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);
1012
+
1013
+ // Set whether the model is in warmup mode or not
1014
+ // If true, all model tensors are activated during llama_decode() to load and cache their weights.
1015
+ //
1016
+ // note: using this can cause extra graph reallocations because it changes the graph topology with MoE models,
1017
+ // so it is generally not recommended to use in practice. will be removed in the future
1018
+ DEPRECATED(LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup),
1019
+ "user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]");
1020
+
1021
+ // Set abort callback
1022
+ LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data);
1023
+
1024
+ // Wait until all computations are finished
1025
+ // This is automatically done when using one of the functions below to obtain the computation results
1026
+ // and is not necessary to call it explicitly in most cases
1027
+ LLAMA_API void llama_synchronize(struct llama_context * ctx);
1028
+
1029
+ // Token logits obtained from the last call to llama_decode()
1030
+ // The logits for which llama_batch.logits[i] != 0 are stored contiguously
1031
+ // in the order they have appeared in the batch.
1032
+ // Rows: number of tokens for which llama_batch.logits[i] != 0
1033
+ // Cols: n_vocab
1034
+ // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)
1035
+ LLAMA_API float * llama_get_logits(struct llama_context * ctx);
1036
+
1037
+ // Logits for the ith token. For positive indices, Equivalent to:
1038
+ // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab
1039
+ // Negative indices can be used to access logits in reverse order, -1 is the last logit.
1040
+ // returns NULL for invalid ids.
1041
+ LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);
1042
+
1043
+ // Get all output token embeddings.
1044
+ // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model,
1045
+ // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously
1046
+ // in the order they have appeared in the batch.
1047
+ // shape: [n_outputs*n_embd]
1048
+ // Otherwise, returns NULL.
1049
+ // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)
1050
+ LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
1051
+
1052
+ // Get the embeddings for the ith token. For positive indices, Equivalent to:
1053
+ // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd
1054
+ // Negative indices can be used to access embeddings in reverse order, -1 is the last embedding.
1055
+ // shape: [n_embd] (1-dimensional)
1056
+ // returns NULL for invalid ids.
1057
+ LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);
1058
+
1059
+ // Get the embeddings for a sequence id
1060
+ // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE
1061
+ // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[n_cls_out] with the rank(s) of the sequence
1062
+ // otherwise: float[n_embd] (1-dimensional)
1063
+ LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id);
1064
+
1065
+ //
1066
+ // backend sampling API [EXPERIMENTAL]
1067
+ // note: use only if the llama_context was created with at least one llama_sampler_seq_config
1068
+ //
1069
+
1070
+ // Get the backend sampled token for the ith token.
1071
+ // With multiple outputs, sampler state advances when the token is accepted,
1072
+ // not when it is read through this function.
1073
+ // When accepting multiple outputs, accept a contiguous prefix in output order.
1074
+ // Returns LLAMA_TOKEN_NULL if no token was sampled.
1075
+ LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i);
1076
+
1077
+ // Get the backend sampled probabilities for the ith token
1078
+ // The index matches llama_get_sampled_token_ith().
1079
+ // Returns NULL if no probabilities were generated.
1080
+ LLAMA_API float * llama_get_sampled_probs_ith (struct llama_context * ctx, int32_t i);
1081
+ LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i);
1082
+
1083
+ // Get the backend sampled logits for the ith token
1084
+ // Returns NULL if no logits were sampled.
1085
+ LLAMA_API float * llama_get_sampled_logits_ith (struct llama_context * ctx, int32_t i);
1086
+ LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i);
1087
+
1088
+ // Get the backend sampled candidates (token ids) for the ith token
1089
+ // These are needed to map probability/logit indices to vocab token ids.
1090
+ // Returns NULL if no candidates were sampled.
1091
+ LLAMA_API llama_token * llama_get_sampled_candidates_ith (struct llama_context * ctx, int32_t i);
1092
+ LLAMA_API uint32_t llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i);
1093
+
1094
+ //
1095
+ // Vocab
1096
+ //
1097
+
1098
+ LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token);
1099
+
1100
+ LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token);
1101
+
1102
+ LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token);
1103
+
1104
+ // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)
1105
+ LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token);
1106
+
1107
+ // Identify if Token Id is a control token or a render-able token
1108
+ LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token);
1109
+
1110
+ // Special tokens
1111
+ LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence
1112
+ LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence
1113
+ LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn
1114
+ LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator
1115
+ LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line
1116
+ LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding
1117
+ LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask
1118
+
1119
+ LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab);
1120
+ LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab);
1121
+ LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab);
1122
+
1123
+ // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens)
1124
+ LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens);
1125
+
1126
+ LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);
1127
+ LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab);
1128
+ LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab);
1129
+ LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab);
1130
+ LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab);
1131
+ LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab);
1132
+
1133
+ DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead");
1134
+ DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead");
1135
+ DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead");
1136
+ DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead");
1137
+ DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead");
1138
+ DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead");
1139
+ DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead");
1140
+ DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead");
1141
+ DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead");
1142
+ DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead");
1143
+ DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead");
1144
+ DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead");
1145
+ DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead");
1146
+ DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead");
1147
+ DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead");
1148
+ DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead");
1149
+ DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead");
1150
+ DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead");
1151
+ DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead");
1152
+ DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead");
1153
+
1154
+ // CLS is equivalent to BOS
1155
+ DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification
1156
+ "use llama_vocab_bos instead");
1157
+
1158
+ //
1159
+ // Tokenization
1160
+ //
1161
+ // The API is thread-safe.
1162
+ //
1163
+
1164
+ /// @details Convert the provided text into tokens.
1165
+ /// @param tokens The tokens pointer must be large enough to hold the resulting tokens.
1166
+ /// @return Returns the number of tokens on success, no more than n_tokens_max
1167
+ /// @return Returns a negative number on failure - the number of tokens that would have been returned
1168
+ /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit)
1169
+ /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so.
1170
+ /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated
1171
+ /// as plaintext. Does not insert a leading space.
1172
+ LLAMA_API int32_t llama_tokenize(
1173
+ const struct llama_vocab * vocab,
1174
+ const char * text,
1175
+ int32_t text_len,
1176
+ llama_token * tokens,
1177
+ int32_t n_tokens_max,
1178
+ bool add_special,
1179
+ bool parse_special);
1180
+
1181
+ // Token Id -> Piece.
1182
+ // Uses the vocabulary in the provided context.
1183
+ // Does not write null terminator to the buffer.
1184
+ // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix')
1185
+ // @param special If true, special tokens are rendered in the output.
1186
+ LLAMA_API int32_t llama_token_to_piece(
1187
+ const struct llama_vocab * vocab,
1188
+ llama_token token,
1189
+ char * buf,
1190
+ int32_t length,
1191
+ int32_t lstrip,
1192
+ bool special);
1193
+
1194
+ /// @details Convert the provided tokens into text (inverse of llama_tokenize()).
1195
+ /// @param text The char pointer must be large enough to hold the resulting text.
1196
+ /// @return Returns the number of chars/bytes on success, no more than text_len_max.
1197
+ /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned.
1198
+ /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so.
1199
+ /// @param unparse_special If true, special tokens are rendered in the output.
1200
+ LLAMA_API int32_t llama_detokenize(
1201
+ const struct llama_vocab * vocab,
1202
+ const llama_token * tokens,
1203
+ int32_t n_tokens,
1204
+ char * text,
1205
+ int32_t text_len_max,
1206
+ bool remove_special,
1207
+ bool unparse_special);
1208
+
1209
+ //
1210
+ // Chat templates
1211
+ //
1212
+
1213
+ /// Apply chat template. Inspired by hf apply_chat_template() on python.
1214
+ ///
1215
+ /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template
1216
+ /// @param tmpl A Jinja template to use for this chat.
1217
+ /// @param chat Pointer to a list of multiple llama_chat_message
1218
+ /// @param n_msg Number of llama_chat_message in this chat
1219
+ /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message.
1220
+ /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages)
1221
+ /// @param length The size of the allocated buffer
1222
+ /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template.
1223
+ LLAMA_API int32_t llama_chat_apply_template(
1224
+ const char * tmpl,
1225
+ const struct llama_chat_message * chat,
1226
+ size_t n_msg,
1227
+ bool add_ass,
1228
+ char * buf,
1229
+ int32_t length);
1230
+
1231
+ // Get list of built-in chat templates
1232
+ LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len);
1233
+
1234
+ //
1235
+ // Sampling API
1236
+ //
1237
+ // Sample usage:
1238
+ //
1239
+ // // prepare the sampling chain at the start
1240
+ // auto sparams = llama_sampler_chain_default_params();
1241
+ //
1242
+ // llama_sampler * smpl = llama_sampler_chain_init(sparams);
1243
+ //
1244
+ // llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50));
1245
+ // llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1));
1246
+ // llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8));
1247
+ //
1248
+ // // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat"
1249
+ // // this sampler will be responsible to select the actual token
1250
+ // llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed));
1251
+ //
1252
+ // ...
1253
+ //
1254
+ // // decoding loop:
1255
+ // while (...) {
1256
+ // ...
1257
+ //
1258
+ // llama_decode(ctx, batch);
1259
+ //
1260
+ // // sample from the logits of the last token in the batch
1261
+ // const llama_token id = llama_sampler_sample(smpl, ctx, -1);
1262
+ //
1263
+ // ...
1264
+ // }
1265
+ //
1266
+ // llama_sampler_free(smpl);
1267
+ //
1268
+
1269
+ typedef void * llama_sampler_context_t;
1270
+
1271
+ struct llama_sampler_data {
1272
+ struct ggml_tensor * logits;
1273
+ struct ggml_tensor * probs;
1274
+ struct ggml_tensor * sampled;
1275
+ struct ggml_tensor * candidates;
1276
+ };
1277
+
1278
+ // user code can implement the interface below in order to create custom llama_sampler
1279
+ struct llama_sampler_i {
1280
+ const char * (*name) (const struct llama_sampler * smpl); // can be NULL
1281
+ void (*accept)( struct llama_sampler * smpl, llama_token token); // can be NULL
1282
+ void (*apply) ( struct llama_sampler * smpl, llama_token_data_array * cur_p); // required
1283
+ void (*reset) ( struct llama_sampler * smpl); // can be NULL
1284
+ struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL
1285
+ void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL
1286
+
1287
+ // [EXPERIMENTAL]
1288
+ // backend sampling interface:
1289
+
1290
+ // return true if the backend supports all ops needed by the sampler and can handle up to n_outputs_max_per_seq outputs per sequence
1291
+ // note: call once per sampler
1292
+ bool (*backend_init)(
1293
+ struct llama_sampler * smpl,
1294
+ ggml_backend_buffer_type_t buft,
1295
+ uint32_t n_outputs_max_per_seq);
1296
+
1297
+ // call after .backend_apply()
1298
+ void (*backend_accept)(
1299
+ struct llama_sampler * smpl,
1300
+ struct ggml_context * ctx,
1301
+ struct ggml_cgraph * gf,
1302
+ struct ggml_tensor * selected_token);
1303
+
1304
+ // call after .backend_init()
1305
+ void (*backend_apply)(
1306
+ struct llama_sampler * smpl,
1307
+ struct ggml_context * ctx,
1308
+ struct ggml_cgraph * gf,
1309
+ struct llama_sampler_data * data);
1310
+
1311
+ // called before graph execution to set inputs for the current ubatch
1312
+ void (*backend_set_input)(struct llama_sampler * smpl);
1313
+
1314
+ // called before rebuilding a sampling graph to clear any internal sampler state
1315
+ void (*backend_reset)(struct llama_sampler * smpl);
1316
+
1317
+ // copy mutable state from src into dst while keeping dst's references to the current sampling graph
1318
+ // src and dst must have the same type and configuration
1319
+ void (*copy_state)(const struct llama_sampler * src, struct llama_sampler * dst);
1320
+ };
1321
+
1322
+ struct llama_sampler {
1323
+ struct llama_sampler_i * iface;
1324
+
1325
+ llama_sampler_context_t ctx;
1326
+ };
1327
+
1328
+ // [EXPERIMENTAL]
1329
+ // attach a sampler to the context
1330
+ // note: prefer initializing the context with llama_context_params.samplers when possible
1331
+ LLAMA_API bool llama_set_sampler(struct llama_context * ctx, llama_seq_id seq_id, struct llama_sampler * smpl);
1332
+
1333
+ // mirror of llama_sampler_i:
1334
+ LLAMA_API struct llama_sampler * llama_sampler_init ( struct llama_sampler_i * iface, llama_sampler_context_t ctx);
1335
+ LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl);
1336
+ LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token);
1337
+ LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p);
1338
+ LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl);
1339
+ LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl);
1340
+ LLAMA_API void llama_sampler_copy (const struct llama_sampler * src, struct llama_sampler * dst);
1341
+ // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add)
1342
+ LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl);
1343
+
1344
+ // llama_sampler_chain
1345
+ // a type of llama_sampler that can chain multiple samplers one after another
1346
+
1347
+ LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params);
1348
+
1349
+ // important: takes ownership of the sampler object and will free it when llama_sampler_free is called
1350
+ LLAMA_API void llama_sampler_chain_add( struct llama_sampler * chain, struct llama_sampler * smpl);
1351
+
1352
+ // return NULL if:
1353
+ // - the sampler is NULL
1354
+ // - the sampler is not a llama_sampler_chain
1355
+ // - the index is out of bounds, unless i == -1
1356
+ // - if i == -1, returns the chain itself (can be used to check if the sampler is a chain)
1357
+ LLAMA_API struct llama_sampler * llama_sampler_chain_get( struct llama_sampler * chain, int32_t i);
1358
+
1359
+ // the total number of samplers in the chain
1360
+ LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain);
1361
+
1362
+ // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed
1363
+ LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i);
1364
+
1365
+ // available samplers:
1366
+
1367
+ LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void);
1368
+
1369
+ /// seed == LLAMA_DEFAULT_SEED to use a random seed.
1370
+ LLAMA_API struct llama_sampler * llama_sampler_init_dist(uint32_t seed);
1371
+
1372
+ /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
1373
+ /// Setting k <= 0 makes this a noop
1374
+ LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k);
1375
+
1376
+ /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751
1377
+ LLAMA_API struct llama_sampler * llama_sampler_init_top_p (float p, size_t min_keep);
1378
+
1379
+ /// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841
1380
+ LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p, size_t min_keep);
1381
+
1382
+ /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.
1383
+ LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep);
1384
+
1385
+ /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf
1386
+ LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t);
1387
+
1388
+ /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772.
1389
+ LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t, float delta, float exponent);
1390
+
1391
+ /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335
1392
+ LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p, float t, size_t min_keep, uint32_t seed);
1393
+
1394
+ /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641
1395
+ LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma(float n);
1396
+
1397
+ /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
1398
+ /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
1399
+ /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
1400
+ /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
1401
+ /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm.
1402
+ /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
1403
+ LLAMA_API struct llama_sampler * llama_sampler_init_mirostat(
1404
+ int32_t n_vocab,
1405
+ uint32_t seed,
1406
+ float tau,
1407
+ float eta,
1408
+ int32_t m);
1409
+
1410
+ /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.
1411
+ /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text.
1412
+ /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.
1413
+ /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.
1414
+ /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal.
1415
+ LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2(
1416
+ uint32_t seed,
1417
+ float tau,
1418
+ float eta);
1419
+
1420
+ /// @details Initializes a GBNF grammar, see grammars/README.md for details.
1421
+ /// @param vocab The vocabulary that this grammar will be used with.
1422
+ /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails.
1423
+ /// @param grammar_root The name of the start symbol for the grammar.
1424
+ LLAMA_API struct llama_sampler * llama_sampler_init_grammar(
1425
+ const struct llama_vocab * vocab,
1426
+ const char * grammar_str,
1427
+ const char * grammar_root);
1428
+
1429
+ DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy(
1430
+ const struct llama_vocab * vocab,
1431
+ const char * grammar_str,
1432
+ const char * grammar_root,
1433
+ const char ** trigger_words,
1434
+ size_t num_trigger_words,
1435
+ const llama_token * trigger_tokens,
1436
+ size_t num_trigger_tokens),
1437
+ "use llama_sampler_init_grammar_lazy_patterns instead");
1438
+
1439
+
1440
+ /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639
1441
+ /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group.
1442
+ /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included.
1443
+ LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns(
1444
+ const struct llama_vocab * vocab,
1445
+ const char * grammar_str,
1446
+ const char * grammar_root,
1447
+ const char ** trigger_patterns,
1448
+ size_t num_trigger_patterns,
1449
+ const llama_token * trigger_tokens,
1450
+ size_t num_trigger_tokens);
1451
+
1452
+
1453
+ /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first.
1454
+ LLAMA_API struct llama_sampler * llama_sampler_init_penalties(
1455
+ int32_t n_vocab,
1456
+ int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty)
1457
+ float penalty_repeat, // must be > 0.0, 1.0 = disabled
1458
+ float penalty_freq, // must be finite, 0.0 = disabled
1459
+ float penalty_present); // must be finite, 0.0 = disabled
1460
+
1461
+ /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982
1462
+ LLAMA_API struct llama_sampler * llama_sampler_init_dry(
1463
+ const struct llama_vocab * vocab,
1464
+ float dry_multiplier,
1465
+ float dry_base,
1466
+ int32_t dry_allowed_length,
1467
+ int32_t dry_penalty_last_n, // last n tokens to penalize (0 = disable penalty)
1468
+ const char ** seq_breakers,
1469
+ size_t num_breakers);
1470
+
1471
+ /// adaptive-p: select tokens near a configurable target probability over time.
1472
+ ///
1473
+ /// the adaptive-p sampler transforms the token probability distribution to favor tokens
1474
+ /// that fall near a user-configurable probability target.
1475
+ ///
1476
+ /// internally, the sampler maintains an exponential moving average of the *ORIGINAL*
1477
+ /// probabilities of selected tokens at each sampling step. it uses this EMA to compute an
1478
+ /// adapted target probability at each sampling step, thus maintaining the desired target
1479
+ /// probability over time.
1480
+ ///
1481
+ /// adaptive-p selects a token ID rather than just mutating candidates, so it must be last
1482
+ /// in the sampler chain (like mirostat, dist, greedy).
1483
+ ///
1484
+ /// only mild truncation before this sampler is recommended. we suggest applying min-p
1485
+ /// before adaptive-p as the only other active sampler in the chain.
1486
+ ///
1487
+ /// @param target select tokens near this probability (valid range 0.0 to 1.0; negative = disabled)
1488
+ /// @param decay EMA decay for adaptation; history ≈ 1/(1-decay) tokens (valid range 0.0 - 0.99)
1489
+ /// @param seed RNG seed
1490
+ ///
1491
+ /// ref: https://github.com/ggml-org/llama.cpp/pull/17927
1492
+ ///
1493
+ LLAMA_API struct llama_sampler * llama_sampler_init_adaptive_p(
1494
+ float target,
1495
+ float decay,
1496
+ uint32_t seed);
1497
+
1498
+ LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias(
1499
+ int32_t n_vocab,
1500
+ int32_t n_logit_bias,
1501
+ const llama_logit_bias * logit_bias);
1502
+
1503
+ // this sampler is meant to be used for fill-in-the-middle infilling
1504
+ // it's supposed to be used after top_k + top_p sampling
1505
+ //
1506
+ // 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG
1507
+ // 2. combine probs of tokens that have the same prefix
1508
+ //
1509
+ // example:
1510
+ //
1511
+ // - before:
1512
+ // "hel": 0.5
1513
+ // "hell": 0.2
1514
+ // "hello": 0.1
1515
+ // "dummy": 0.1
1516
+ //
1517
+ // - after:
1518
+ // "hel": 0.8
1519
+ // "dummy": 0.1
1520
+ //
1521
+ // 3. discard non-EOG tokens with low prob
1522
+ // 4. if no tokens are left -> pick EOT
1523
+ //
1524
+ LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab);
1525
+
1526
+ // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise
1527
+ LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl);
1528
+
1529
+ /// @details Sample and accept a token from the idx-th output of the last evaluation
1530
+ // For multiple outputs from one sampler, call this function in output order without gaps.
1531
+ //
1532
+ // Shorthand for:
1533
+ // const auto * logits = llama_get_logits_ith(ctx, idx);
1534
+ // llama_token_data_array cur_p = { ... init from logits ... };
1535
+ // llama_sampler_apply(smpl, &cur_p);
1536
+ // auto token = cur_p.data[cur_p.selected].id;
1537
+ // llama_sampler_accept(smpl, token);
1538
+ // return token;
1539
+ // Returns the sampled token
1540
+ LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx);
1541
+
1542
+ // TODO: extend in the future
1543
+ //LLAMA_API void llama_decode_with_sampler(struct llama_context * ctx, struct llama_sampler * smpl, struct llama_batch batch, ...);
1544
+
1545
+ //
1546
+ // Model split
1547
+ //
1548
+
1549
+ /// @details Build a split GGUF final path for this chunk.
1550
+ /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf"
1551
+ // Returns the split_path length.
1552
+ LLAMA_API int32_t llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int32_t split_no, int32_t split_count);
1553
+
1554
+ /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match.
1555
+ /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0"
1556
+ // Returns the split_prefix length.
1557
+ LLAMA_API int32_t llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int32_t split_no, int32_t split_count);
1558
+
1559
+ // Print system information
1560
+ LLAMA_API const char * llama_print_system_info(void);
1561
+
1562
+ // Set callback for all future logging events.
1563
+ // If this is not called, or NULL is supplied, everything is output on stderr.
1564
+ // The logger state is global so these functions are NOT thread safe.
1565
+ LLAMA_API void llama_log_get(ggml_log_callback * log_callback, void ** user_data);
1566
+ LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data);
1567
+
1568
+ //
1569
+ // Performance utils
1570
+ //
1571
+ // NOTE: Used by llama.cpp examples/tools, avoid using in third-party apps. Instead, do your own performance measurements.
1572
+ //
1573
+
1574
+ struct llama_perf_context_data {
1575
+ // ms == milliseconds
1576
+ double t_start_ms; // absolute start time
1577
+ double t_load_ms; // time needed for loading the model
1578
+ double t_p_eval_ms; // time needed for processing the prompt
1579
+ double t_eval_ms; // time needed for generating tokens
1580
+
1581
+ int32_t n_p_eval; // number of prompt tokens
1582
+ int32_t n_eval; // number of generated tokens
1583
+ int32_t n_reused; // number of times a ggml compute graph had been reused
1584
+ };
1585
+
1586
+ struct llama_perf_sampler_data {
1587
+ double t_sample_ms; // time needed for sampling in ms
1588
+
1589
+ int32_t n_sample; // number of sampled tokens
1590
+ };
1591
+
1592
+ LLAMA_API struct llama_perf_context_data llama_perf_context (const struct llama_context * ctx);
1593
+ LLAMA_API void llama_perf_context_print(const struct llama_context * ctx);
1594
+ LLAMA_API void llama_perf_context_reset( struct llama_context * ctx);
1595
+
1596
+ // NOTE: the following work only with samplers constructed via llama_sampler_chain_init
1597
+ LLAMA_API struct llama_perf_sampler_data llama_perf_sampler (const struct llama_sampler * chain);
1598
+ LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain);
1599
+ LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain);
1600
+
1601
+ //
1602
+ // training
1603
+ //
1604
+
1605
+ // function that returns whether or not a given tensor contains trainable parameters
1606
+ typedef bool (*llama_opt_param_filter)(const struct ggml_tensor * tensor, void * userdata);
1607
+
1608
+ // always returns true
1609
+ LLAMA_API bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata);
1610
+
1611
+ struct llama_opt_params {
1612
+ uint32_t n_ctx_train; // assumed context size post training, use context size specified in llama_context if 0
1613
+
1614
+ llama_opt_param_filter param_filter; // callback for determining which tensors contain trainable parameters
1615
+ void * param_filter_ud; // userdata for determining which tensors contain trainable parameters
1616
+
1617
+ ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters
1618
+ void * get_opt_pars_ud; // userdata for calculating optimizer parameters
1619
+
1620
+ enum ggml_opt_optimizer_type optimizer_type;
1621
+ };
1622
+
1623
+ LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params);
1624
+
1625
+ LLAMA_API void llama_opt_epoch(
1626
+ struct llama_context * lctx,
1627
+ ggml_opt_dataset_t dataset,
1628
+ ggml_opt_result_t result_train,
1629
+ ggml_opt_result_t result_eval,
1630
+ int64_t idata_split,
1631
+ ggml_opt_epoch_callback callback_train,
1632
+ ggml_opt_epoch_callback callback_eval);
1633
+
1634
+ #ifdef __cplusplus
1635
+ }
1636
+ #endif
1637
+
1638
+ #endif // LLAMA_H
licenses/LICENSE-jsonhpp ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2013-2025 Niels Lohmann
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
media/llama0-banner.png ADDED

Git LFS Details

  • SHA256: f15963f0d41f76ff4fb341d919c5dd0e01c22d8c839935d0ef390ea968ab7a1c
  • Pointer size: 131 Bytes
  • Size of remote file: 145 kB
media/llama0-logo.png ADDED

Git LFS Details

  • SHA256: 4aa1256ae1b68367e8ad32d4f47d8b69c29be35a4f2c01c7fa3f45f9fdae16f8
  • Pointer size: 131 Bytes
  • Size of remote file: 180 kB
media/llama1-banner.png ADDED
media/llama1-icon-transparent.png ADDED
media/llama1-icon-transparent.svg ADDED
media/llama1-icon.png ADDED
media/llama1-icon.svg ADDED
media/llama1-logo.png ADDED
media/llama1-logo.svg ADDED
media/matmul.png ADDED

Git LFS Details

  • SHA256: 0d6484cc146bbfcc1e8eae12ed0290057bfabeb7944d24d15d5d78995364bde4
  • Pointer size: 131 Bytes
  • Size of remote file: 266 kB
media/matmul.svg ADDED
mypy.ini ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ [mypy]
2
+ strict = true
3
+ allow_untyped_calls = true
4
+ allow_untyped_defs = true
5
+ allow_incomplete_defs = true
6
+ disable_error_code = import-untyped
7
+ warn_return_any = false
out_base.txt ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ user
2
+ Explain how photosynthesis works in plants.
3
+ assistant
4
+ Photosynthesis is a process that plants use to convert sunlight into energy. It occurs primarily in the chlorophyll-containing cells of plants (especially in leaves), and it involves three main stages: light absorption, chemical reactions, and energy production.
5
+
6
+ Here’s how photosynthesis works step by step:
7
+
8
+ 1. **Light Absorption (Photosynthesis in the Light Reactions):**
9
+ - Plants absorb sunlight through their leaves, especially through chlorophyll.
10
+ - Light energy is captured by the chlorophyll molecules in the thylakoid membranes of the chloroplasts.
11
+ - The sunlight (photons) excites electrons in the chlorophyll, which leads to the splitting of water molecules (H₂O) into oxygen (O₂), hydrogen (H⁺), and electrons.
12
+
13
+ 2. **Chemical Reactions (Calvin Cycle):**
14
+ - The energy from sunlight is used to produce ATP (adenosine triphosphate) and NADPH (nicotinamide dinuclephosphate).
15
+ - These molecules are used in the Calvin Cycle (dark reactions).
16
+ - Carbon dioxide (CO₂) is fixed into sugar molecules (like glucose) through a series of reactions.
17
+ - The process begins with CO₂ being converted into glucose using ATP and
18
+ > EOF by user
19
+
20
+
out_cpu.txt ADDED
File without changes
out_cuda.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ user
2
+ The ocean is
3
+ assistant
4
+ The ocean is vast, mysterious, and full of life. It covers about 71% of the Earth's surface and plays a crucial role in regulating climate, supporting ecosystems, and sustaining human activities
5
+ > EOF by user
6
+
7
+
pocs/CMakeLists.txt ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # dependencies
2
+
3
+ find_package(Threads REQUIRED)
4
+
5
+ # third-party
6
+
7
+ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
8
+
9
+ if (EMSCRIPTEN)
10
+ else()
11
+ if (NOT GGML_BACKEND_DL)
12
+ add_subdirectory(vdot)
13
+ endif()
14
+ endif()
pocs/vdot/CMakeLists.txt ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ set(TARGET llama-vdot)
2
+ add_executable(${TARGET} vdot.cpp)
3
+ target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT})
4
+ target_compile_features(${TARGET} PRIVATE cxx_std_17)
5
+
6
+ set(TARGET llama-q8dot)
7
+ add_executable(${TARGET} q8dot.cpp)
8
+ target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT})
9
+ target_compile_features(${TARGET} PRIVATE cxx_std_17)
pocs/vdot/q8dot.cpp ADDED
@@ -0,0 +1,173 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include <cstdio>
2
+ #include <type_traits>
3
+ #include <vector>
4
+ #include <random>
5
+ #include <chrono>
6
+ #include <cstdlib>
7
+ #include <cmath>
8
+ #include <cassert>
9
+ #include <cstring>
10
+ #include <array>
11
+ #include <type_traits>
12
+
13
+ #include <ggml.h>
14
+ #include <ggml-cpu.h>
15
+
16
+ constexpr int kVecSize = 1 << 16;
17
+
18
+ // Copy-pasted from ggml.c
19
+ #define QK4_0 32
20
+ typedef struct {
21
+ float d; // delta
22
+ uint8_t qs[QK4_0 / 2]; // nibbles / quants
23
+ } block_q4_0;
24
+ static_assert(sizeof(block_q4_0) == sizeof(float) + QK4_0 / 2, "wrong q4_0 block size/padding");
25
+
26
+ #define QK4_1 32
27
+ typedef struct {
28
+ float d; // delta
29
+ float m; // min
30
+ uint8_t qs[QK4_1 / 2]; // nibbles / quants
31
+ } block_q4_1;
32
+ static_assert(sizeof(block_q4_1) == sizeof(float) * 2 + QK4_1 / 2, "wrong q4_1 block size/padding");
33
+
34
+ // Copy-pasted from ggml.c
35
+ #define QK8_0 32
36
+ typedef struct {
37
+ float d; // delta
38
+ float s; // d * sum(qs[i])
39
+ int8_t qs[QK8_0]; // quants
40
+ } block_q8_0;
41
+ static_assert(sizeof(block_q8_0) == 2*sizeof(float) + QK8_0, "wrong q8_0 block size/padding");
42
+
43
+ static_assert(QK4_1 == QK8_0, "QK4_1 and QK8_0 must be the same");
44
+ static_assert(QK4_0 == QK8_0, "QK4_0 and QK8_0 must be the same");
45
+
46
+ template <typename T>
47
+ static void fillQ4blocks(std::vector<T>& blocks, std::mt19937& rndm) {
48
+ for (auto& b : blocks) {
49
+ b.d = 1;
50
+ for (int i=0; i<QK4_1/2; ++i) {
51
+ uint8_t v1 = rndm() >> 28;
52
+ uint8_t v2 = rndm() >> 28;
53
+ b.qs[i] = v1 | (v2 << 4);
54
+ }
55
+ }
56
+ }
57
+
58
+ static void fillQ80blocks(std::vector<block_q8_0>& blocks, std::mt19937& rndm) {
59
+ for (auto& b : blocks) {
60
+ b.d = 1;
61
+ int sum = 0;
62
+ for (int i=0; i<QK8_0; ++i) {
63
+ b.qs[i] = (rndm() >> 24) - 128;
64
+ sum += b.qs[i];
65
+ }
66
+ b.s = b.d * sum;
67
+ }
68
+ }
69
+
70
+ static float simpleDot(const block_q4_0& x, const block_q8_0& y) {
71
+ int s1 = 0; //, s2 = 0;
72
+ for (int i=0; i<QK4_1/2; i+=2) {
73
+ int v1 = x.qs[i+0] & 0xf;
74
+ int v2 = x.qs[i+0] >> 4;
75
+ int v3 = x.qs[i+1] & 0xf;
76
+ int v4 = x.qs[i+1] >> 4;
77
+ int j = 2*i;
78
+ s1 += v1*y.qs[j] + v2*y.qs[j+1] + v3*y.qs[j+2] + v4*y.qs[j+3];
79
+ //s2 += y.qs[j] + y.qs[j+1] + y.qs[j+2] + y.qs[j+3];
80
+ }
81
+ return y.d * x.d * s1 - 8 * x.d * y.s;
82
+ //return y.d * x.d * (s1 - 8 * s2);
83
+ }
84
+
85
+ static float simpleDot(const block_q4_1& x, const block_q8_0& y) {
86
+ int s1 = 0; //, s2 = 0;
87
+ for (int i=0; i<QK4_1/2; i+=2) {
88
+ int v1 = x.qs[i+0] & 0xf;
89
+ int v2 = x.qs[i+0] >> 4;
90
+ int v3 = x.qs[i+1] & 0xf;
91
+ int v4 = x.qs[i+1] >> 4;
92
+ int j = 2*i;
93
+ s1 += v1*y.qs[j] + v2*y.qs[j+1] + v3*y.qs[j+2] + v4*y.qs[j+3];
94
+ //s2 += y.qs[j] + y.qs[j+1] + y.qs[j+2] + y.qs[j+3];
95
+ }
96
+ return y.d * x.d * s1 + y.s * x.m;
97
+ //return y.d * (x.d * s1 + x.m * s2);
98
+ }
99
+
100
+ struct Stat {
101
+ double sum = 0, sumt = 0, sumt2 = 0, maxt = 0;
102
+ int nloop = 0;
103
+ void addResult(double s, double t) {
104
+ sum += s;
105
+ sumt += t; sumt2 += t*t; maxt = std::max(maxt, t);
106
+ ++nloop;
107
+ }
108
+ void reportResult(const char* title) const {
109
+ if (nloop < 1) {
110
+ printf("%s(%s): no result\n",__func__,title);
111
+ return;
112
+ }
113
+ printf("============ %s\n",title);
114
+ printf("<dot> = %g\n",sum/nloop);
115
+ auto t = sumt/nloop, dt = sumt2/nloop - t*t;
116
+ if (dt > 0) dt = sqrt(dt);
117
+ printf("<time> = %g +/- %g us. Max. time = %g us.\n",t,dt,maxt);
118
+ }
119
+ };
120
+
121
+
122
+ int main(int argc, char** argv) {
123
+
124
+ int nloop = argc > 1 ? atoi(argv[1]) : 10;
125
+ int type = argc > 2 ? atoi(argv[2]) : 1;
126
+
127
+ std::mt19937 rndm(1234);
128
+
129
+ std::vector<block_q4_1> x41;
130
+ std::vector<block_q4_0> x40;
131
+ std::vector<block_q8_0> y(kVecSize);
132
+ if (type == 0) x40.resize(kVecSize);
133
+ else {
134
+ x41.resize(kVecSize);
135
+ for (auto& b : x41) b.m = 1;
136
+ }
137
+
138
+ auto ggml_type = type == 0 ? GGML_TYPE_Q4_0 : GGML_TYPE_Q4_1;
139
+
140
+ const auto * funcs = ggml_get_type_traits_cpu(ggml_type);
141
+
142
+ Stat simple, ggml;
143
+
144
+ for (int iloop=0; iloop<nloop; ++iloop) {
145
+
146
+ if (type == 0) fillQ4blocks(x40, rndm);
147
+ else fillQ4blocks(x41, rndm);
148
+ fillQ80blocks(y, rndm);
149
+
150
+ auto t1 = std::chrono::high_resolution_clock::now();
151
+ double s = 0;
152
+ if (type == 0) for (int i=0; i<kVecSize; ++i) s += simpleDot(x40[i], y[i]);
153
+ else for (int i=0; i<kVecSize; ++i) s += simpleDot(x41[i], y[i]);
154
+ auto t2 = std::chrono::high_resolution_clock::now();
155
+ auto t = 1e-3*std::chrono::duration_cast<std::chrono::nanoseconds>(t2-t1).count();
156
+ if (iloop > 3) simple.addResult(s, t);
157
+
158
+ t1 = std::chrono::high_resolution_clock::now();
159
+ float fs;
160
+ if (type == 0) funcs->vec_dot(kVecSize * QK4_1, &fs, 0, x40.data(), 0, y.data(), 0, 1);
161
+ else funcs->vec_dot(kVecSize * QK4_1, &fs, 0, x41.data(), 0, y.data(), 0, 1);
162
+ t2 = std::chrono::high_resolution_clock::now();
163
+ t = 1e-3*std::chrono::duration_cast<std::chrono::nanoseconds>(t2-t1).count();
164
+ if (iloop > 3) ggml.addResult(fs, t);
165
+
166
+ }
167
+
168
+ // Report the time (and the average of the dot products so the compiler does not come up with the idea
169
+ // of optimizing away the function calls after figuring that the result is not used).
170
+ simple.reportResult("Simple");
171
+ ggml.reportResult("ggml");
172
+ return 0;
173
+ }
pocs/vdot/vdot.cpp ADDED
@@ -0,0 +1,311 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include <cstdio>
2
+ #include <vector>
3
+ #include <random>
4
+ #include <chrono>
5
+ #include <cstdlib>
6
+ #include <cmath>
7
+ #include <cassert>
8
+ #include <cstring>
9
+ #include <array>
10
+
11
+ #include <ggml.h>
12
+ #include <ggml-cpu.h>
13
+
14
+ #if defined(_MSC_VER)
15
+ #pragma warning(disable: 4244 4267) // possible loss of data
16
+ #endif
17
+
18
+ constexpr int kVecSize = 1 << 18;
19
+
20
+ static float drawFromGaussianPdf(std::mt19937& rndm) {
21
+ constexpr double kScale = 1./(1. + std::mt19937::max());
22
+ constexpr double kTwoPiTimesScale = 6.28318530717958647692*kScale;
23
+ static float lastX;
24
+ static bool haveX = false;
25
+ if (haveX) { haveX = false; return lastX; }
26
+ auto r = sqrt(-2*log(1 - kScale*rndm()));
27
+ auto phi = kTwoPiTimesScale * rndm();
28
+ lastX = r*sin(phi);
29
+ haveX = true;
30
+ return r*cos(phi);
31
+ }
32
+
33
+ static void fillRandomGaussianFloats(std::vector<float>& values, std::mt19937& rndm, float mean = 0) {
34
+ for (auto& v : values) v = mean + drawFromGaussianPdf(rndm);
35
+ }
36
+
37
+ // Copy-pasted from ggml.c
38
+ #define QK4_0 32
39
+ typedef struct {
40
+ float d; // delta
41
+ uint8_t qs[QK4_0 / 2]; // nibbles / quants
42
+ } block_q4_0;
43
+ static_assert(sizeof(block_q4_0) == sizeof(float) + QK4_0 / 2, "wrong q4_0 block size/padding");
44
+
45
+ #define QK4_1 32
46
+ typedef struct {
47
+ float d; // delta
48
+ float m; // min
49
+ uint8_t qs[QK4_1 / 2]; // nibbles / quants
50
+ } block_q4_1;
51
+ static_assert(sizeof(block_q4_1) == sizeof(float) * 2 + QK4_1 / 2, "wrong q4_1 block size/padding");
52
+
53
+ // Copy-pasted from ggml.c
54
+ #define QK8_0 32
55
+ typedef struct {
56
+ float d; // delta
57
+ int8_t qs[QK8_0]; // quants
58
+ } block_q8_0;
59
+ static_assert(sizeof(block_q8_0) == sizeof(float) + QK8_0, "wrong q8_0 block size/padding");
60
+
61
+ // "Scalar" dot product between the quantized vector x and float vector y
62
+ inline double dot(int n, const block_q4_0* x, const float* y) {
63
+ const static float kValues[16] = {-8.f, -7.f, -6.f, -5.f, -4.f, -3.f, -2.f, -1.f, 0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f};
64
+ constexpr uint32_t kMask1 = 0x0f0f0f0f;
65
+ uint32_t u1, u2;
66
+ auto q1 = (const uint8_t*)&u1;
67
+ auto q2 = (const uint8_t*)&u2;
68
+ double sum = 0;
69
+ for (int i=0; i<n; ++i) {
70
+ float d = x->d;
71
+ auto u = (const uint32_t*)x->qs;
72
+ float s = 0;
73
+ for (int k=0; k<4; ++k) {
74
+ u1 = u[k] & kMask1;
75
+ u2 = (u[k] >> 4) & kMask1;
76
+ s += y[0]*kValues[q1[0]] + y[1]*kValues[q2[0]] +
77
+ y[2]*kValues[q1[1]] + y[3]*kValues[q2[1]] +
78
+ y[4]*kValues[q1[2]] + y[5]*kValues[q2[2]] +
79
+ y[6]*kValues[q1[3]] + y[7]*kValues[q2[3]];
80
+ y += 8;
81
+ }
82
+ sum += s*d;
83
+ ++x;
84
+ }
85
+ return sum;
86
+ }
87
+ // Alternative version of the above. Faster on my Mac (~45 us vs ~55 us per dot product),
88
+ // but about the same on X86_64 (Ryzen 7950X CPU).
89
+ inline double dot3(int n, const block_q4_0* x, const float* y) {
90
+ const static std::pair<float,float> kValues[256] = {
91
+ {-8.f, -8.f}, {-7.f, -8.f}, {-6.f, -8.f}, {-5.f, -8.f}, {-4.f, -8.f}, {-3.f, -8.f}, {-2.f, -8.f}, {-1.f, -8.f},
92
+ { 0.f, -8.f}, { 1.f, -8.f}, { 2.f, -8.f}, { 3.f, -8.f}, { 4.f, -8.f}, { 5.f, -8.f}, { 6.f, -8.f}, { 7.f, -8.f},
93
+ {-8.f, -7.f}, {-7.f, -7.f}, {-6.f, -7.f}, {-5.f, -7.f}, {-4.f, -7.f}, {-3.f, -7.f}, {-2.f, -7.f}, {-1.f, -7.f},
94
+ { 0.f, -7.f}, { 1.f, -7.f}, { 2.f, -7.f}, { 3.f, -7.f}, { 4.f, -7.f}, { 5.f, -7.f}, { 6.f, -7.f}, { 7.f, -7.f},
95
+ {-8.f, -6.f}, {-7.f, -6.f}, {-6.f, -6.f}, {-5.f, -6.f}, {-4.f, -6.f}, {-3.f, -6.f}, {-2.f, -6.f}, {-1.f, -6.f},
96
+ { 0.f, -6.f}, { 1.f, -6.f}, { 2.f, -6.f}, { 3.f, -6.f}, { 4.f, -6.f}, { 5.f, -6.f}, { 6.f, -6.f}, { 7.f, -6.f},
97
+ {-8.f, -5.f}, {-7.f, -5.f}, {-6.f, -5.f}, {-5.f, -5.f}, {-4.f, -5.f}, {-3.f, -5.f}, {-2.f, -5.f}, {-1.f, -5.f},
98
+ { 0.f, -5.f}, { 1.f, -5.f}, { 2.f, -5.f}, { 3.f, -5.f}, { 4.f, -5.f}, { 5.f, -5.f}, { 6.f, -5.f}, { 7.f, -5.f},
99
+ {-8.f, -4.f}, {-7.f, -4.f}, {-6.f, -4.f}, {-5.f, -4.f}, {-4.f, -4.f}, {-3.f, -4.f}, {-2.f, -4.f}, {-1.f, -4.f},
100
+ { 0.f, -4.f}, { 1.f, -4.f}, { 2.f, -4.f}, { 3.f, -4.f}, { 4.f, -4.f}, { 5.f, -4.f}, { 6.f, -4.f}, { 7.f, -4.f},
101
+ {-8.f, -3.f}, {-7.f, -3.f}, {-6.f, -3.f}, {-5.f, -3.f}, {-4.f, -3.f}, {-3.f, -3.f}, {-2.f, -3.f}, {-1.f, -3.f},
102
+ { 0.f, -3.f}, { 1.f, -3.f}, { 2.f, -3.f}, { 3.f, -3.f}, { 4.f, -3.f}, { 5.f, -3.f}, { 6.f, -3.f}, { 7.f, -3.f},
103
+ {-8.f, -2.f}, {-7.f, -2.f}, {-6.f, -2.f}, {-5.f, -2.f}, {-4.f, -2.f}, {-3.f, -2.f}, {-2.f, -2.f}, {-1.f, -2.f},
104
+ { 0.f, -2.f}, { 1.f, -2.f}, { 2.f, -2.f}, { 3.f, -2.f}, { 4.f, -2.f}, { 5.f, -2.f}, { 6.f, -2.f}, { 7.f, -2.f},
105
+ {-8.f, -1.f}, {-7.f, -1.f}, {-6.f, -1.f}, {-5.f, -1.f}, {-4.f, -1.f}, {-3.f, -1.f}, {-2.f, -1.f}, {-1.f, -1.f},
106
+ { 0.f, -1.f}, { 1.f, -1.f}, { 2.f, -1.f}, { 3.f, -1.f}, { 4.f, -1.f}, { 5.f, -1.f}, { 6.f, -1.f}, { 7.f, -1.f},
107
+ {-8.f, 0.f}, {-7.f, 0.f}, {-6.f, 0.f}, {-5.f, 0.f}, {-4.f, 0.f}, {-3.f, 0.f}, {-2.f, 0.f}, {-1.f, 0.f},
108
+ { 0.f, 0.f}, { 1.f, 0.f}, { 2.f, 0.f}, { 3.f, 0.f}, { 4.f, 0.f}, { 5.f, 0.f}, { 6.f, 0.f}, { 7.f, 0.f},
109
+ {-8.f, 1.f}, {-7.f, 1.f}, {-6.f, 1.f}, {-5.f, 1.f}, {-4.f, 1.f}, {-3.f, 1.f}, {-2.f, 1.f}, {-1.f, 1.f},
110
+ { 0.f, 1.f}, { 1.f, 1.f}, { 2.f, 1.f}, { 3.f, 1.f}, { 4.f, 1.f}, { 5.f, 1.f}, { 6.f, 1.f}, { 7.f, 1.f},
111
+ {-8.f, 2.f}, {-7.f, 2.f}, {-6.f, 2.f}, {-5.f, 2.f}, {-4.f, 2.f}, {-3.f, 2.f}, {-2.f, 2.f}, {-1.f, 2.f},
112
+ { 0.f, 2.f}, { 1.f, 2.f}, { 2.f, 2.f}, { 3.f, 2.f}, { 4.f, 2.f}, { 5.f, 2.f}, { 6.f, 2.f}, { 7.f, 2.f},
113
+ {-8.f, 3.f}, {-7.f, 3.f}, {-6.f, 3.f}, {-5.f, 3.f}, {-4.f, 3.f}, {-3.f, 3.f}, {-2.f, 3.f}, {-1.f, 3.f},
114
+ { 0.f, 3.f}, { 1.f, 3.f}, { 2.f, 3.f}, { 3.f, 3.f}, { 4.f, 3.f}, { 5.f, 3.f}, { 6.f, 3.f}, { 7.f, 3.f},
115
+ {-8.f, 4.f}, {-7.f, 4.f}, {-6.f, 4.f}, {-5.f, 4.f}, {-4.f, 4.f}, {-3.f, 4.f}, {-2.f, 4.f}, {-1.f, 4.f},
116
+ { 0.f, 4.f}, { 1.f, 4.f}, { 2.f, 4.f}, { 3.f, 4.f}, { 4.f, 4.f}, { 5.f, 4.f}, { 6.f, 4.f}, { 7.f, 4.f},
117
+ {-8.f, 5.f}, {-7.f, 5.f}, {-6.f, 5.f}, {-5.f, 5.f}, {-4.f, 5.f}, {-3.f, 5.f}, {-2.f, 5.f}, {-1.f, 5.f},
118
+ { 0.f, 5.f}, { 1.f, 5.f}, { 2.f, 5.f}, { 3.f, 5.f}, { 4.f, 5.f}, { 5.f, 5.f}, { 6.f, 5.f}, { 7.f, 5.f},
119
+ {-8.f, 6.f}, {-7.f, 6.f}, {-6.f, 6.f}, {-5.f, 6.f}, {-4.f, 6.f}, {-3.f, 6.f}, {-2.f, 6.f}, {-1.f, 6.f},
120
+ { 0.f, 6.f}, { 1.f, 6.f}, { 2.f, 6.f}, { 3.f, 6.f}, { 4.f, 6.f}, { 5.f, 6.f}, { 6.f, 6.f}, { 7.f, 6.f},
121
+ {-8.f, 7.f}, {-7.f, 7.f}, {-6.f, 7.f}, {-5.f, 7.f}, {-4.f, 7.f}, {-3.f, 7.f}, {-2.f, 7.f}, {-1.f, 7.f},
122
+ { 0.f, 7.f}, { 1.f, 7.f}, { 2.f, 7.f}, { 3.f, 7.f}, { 4.f, 7.f}, { 5.f, 7.f}, { 6.f, 7.f}, { 7.f, 7.f}
123
+ };
124
+ double sum = 0;
125
+ for (int i=0; i<n; ++i) {
126
+ float d = x->d;
127
+ auto q = x->qs;
128
+ float s = 0;
129
+ for (int k=0; k<4; ++k) {
130
+ s += y[0]*kValues[q[0]].first + y[1]*kValues[q[0]].second +
131
+ y[2]*kValues[q[1]].first + y[3]*kValues[q[1]].second +
132
+ y[4]*kValues[q[2]].first + y[5]*kValues[q[2]].second +
133
+ y[6]*kValues[q[3]].first + y[7]*kValues[q[3]].second;
134
+ y += 8; q += 4;
135
+ }
136
+ sum += s*d;
137
+ ++x;
138
+ }
139
+ return sum;
140
+ }
141
+
142
+ inline double dot41(int n, const block_q4_1* x, const float* y) {
143
+ const static float kValues[16] = {0.f, 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f, 9.f, 10.f, 11.f, 12.f, 13.f, 14.f, 15.f};
144
+ constexpr uint32_t kMask1 = 0x0f0f0f0f;
145
+ uint32_t u1, u2;
146
+ auto q1 = (const uint8_t*)&u1;
147
+ auto q2 = (const uint8_t*)&u2;
148
+ double sum = 0;
149
+ for (int i=0; i<n; ++i) {
150
+ auto u = (const uint32_t*)x->qs;
151
+ float s = 0, s1 = 0;
152
+ for (int k=0; k<4; ++k) {
153
+ u1 = u[k] & kMask1;
154
+ u2 = (u[k] >> 4) & kMask1;
155
+ s += y[0]*kValues[q1[0]] + y[1]*kValues[q2[0]] +
156
+ y[2]*kValues[q1[1]] + y[3]*kValues[q2[1]] +
157
+ y[4]*kValues[q1[2]] + y[5]*kValues[q2[2]] +
158
+ y[6]*kValues[q1[3]] + y[7]*kValues[q2[3]];
159
+ s1 += y[0] + y[1] + y[2] + y[3] + y[4] + y[5] + y[6] + y[7];
160
+ y += 8;
161
+ }
162
+ sum += s*x->d + s1*x->m;
163
+ ++x;
164
+ }
165
+ return sum;
166
+ }
167
+
168
+ // Copy-pasted from ggml.c
169
+ static void quantize_row_q8_0_reference(const float *x, block_q8_0 *y, int k) {
170
+ assert(k % QK8_0 == 0);
171
+ const int nb = k / QK8_0;
172
+
173
+ for (int i = 0; i < nb; i++) {
174
+ float amax = 0.0f; // absolute max
175
+
176
+ for (int l = 0; l < QK8_0; l++) {
177
+ const float v = x[i*QK8_0 + l];
178
+ amax = std::max(amax, fabsf(v));
179
+ }
180
+
181
+ const float d = amax / ((1 << 7) - 1);
182
+ const float id = d ? 1.0f/d : 0.0f;
183
+
184
+ y[i].d = d;
185
+
186
+ for (int l = 0; l < QK8_0; ++l) {
187
+ const float v = x[i*QK8_0 + l]*id;
188
+ y[i].qs[l] = roundf(v);
189
+ }
190
+ }
191
+ }
192
+
193
+ // Copy-pasted from ggml.c
194
+ static void dot_q4_q8(const int n, float* s, const void* vx, const void* vy) {
195
+ const int nb = n / QK8_0;
196
+ const block_q4_0* x = (const block_q4_0*)vx;
197
+ const block_q8_0* y = (const block_q8_0*)vy;
198
+ float sumf = 0;
199
+ for (int i = 0; i < nb; i++) {
200
+ const float d0 = x[i].d;
201
+ const float d1 = y[i].d;
202
+
203
+ const uint8_t * p0 = x[i].qs;
204
+ const int8_t * p1 = y[i].qs;
205
+
206
+ int sumi = 0;
207
+ for (int j = 0; j < QK8_0/2; j++) {
208
+ const uint8_t v0 = p0[j];
209
+
210
+ const int i0 = (int8_t) (v0 & 0xf) - 8;
211
+ const int i1 = (int8_t) (v0 >> 4) - 8;
212
+
213
+ const int i2 = p1[2*j + 0];
214
+ const int i3 = p1[2*j + 1];
215
+
216
+ sumi += i0*i2 + i1*i3;
217
+ }
218
+ sumf += d0*d1*sumi;
219
+ }
220
+ *s = sumf;
221
+ }
222
+
223
+ int main(int argc, char** argv) {
224
+
225
+ int nloop = argc > 1 ? atoi(argv[1]) : 10;
226
+ bool scalar = argc > 2 ? atoi(argv[2]) : false;
227
+ bool useQ4_1 = argc > 3 ? atoi(argv[3]) : false;
228
+
229
+ if (scalar && useQ4_1) {
230
+ printf("It is not possible to use Q4_1 quantization and scalar implementations\n");
231
+ return 1;
232
+ }
233
+
234
+ std::mt19937 rndm(1234);
235
+
236
+ std::vector<float> x1(kVecSize), y1(kVecSize);
237
+ int n4 = useQ4_1 ? kVecSize / QK4_1 : kVecSize / QK4_0; n4 = 64*((n4 + 63)/64);
238
+ int n8 = kVecSize / QK8_0; n8 = 64*((n8 + 63)/64);
239
+
240
+ const auto * funcs_cpu = ggml_get_type_traits_cpu(useQ4_1 ? GGML_TYPE_Q4_1 : GGML_TYPE_Q4_0);
241
+
242
+ std::vector<block_q4_0> q40;
243
+ std::vector<block_q4_1> q41;
244
+ if (useQ4_1) q41.resize(n4);
245
+ else q40.resize(n4);
246
+ std::vector<block_q8_0> q8(n8);
247
+ double sumt = 0, sumt2 = 0, maxt = 0;
248
+ double sumqt = 0, sumqt2 = 0, maxqt = 0;
249
+ double sum = 0, sumq = 0, exactSum = 0;
250
+ for (int iloop=0; iloop<nloop; ++iloop) {
251
+
252
+ // Fill vector x with random numbers
253
+ fillRandomGaussianFloats(x1, rndm);
254
+
255
+ // Fill vector y with random numbers
256
+ fillRandomGaussianFloats(y1, rndm);
257
+
258
+ // Compute the exact dot product
259
+ for (int k=0; k<kVecSize; ++k) exactSum += x1[k]*y1[k];
260
+
261
+ // quantize x.
262
+ // Note, we do not include this in the timing as in practical application
263
+ // we already have the quantized model weights.
264
+ if (useQ4_1) {
265
+ funcs_cpu->from_float(x1.data(), q41.data(), kVecSize);
266
+ } else {
267
+ funcs_cpu->from_float(x1.data(), q40.data(), kVecSize);
268
+ }
269
+
270
+ // Now measure time the dot product needs using the "scalar" version above
271
+ auto t1 = std::chrono::high_resolution_clock::now();
272
+ if (useQ4_1) sum += dot41(kVecSize / QK4_1, q41.data(), y1.data());
273
+ else sum += dot(kVecSize / QK4_0, q40.data(), y1.data());
274
+ auto t2 = std::chrono::high_resolution_clock::now();
275
+ auto t = 1e-3*std::chrono::duration_cast<std::chrono::nanoseconds>(t2-t1).count();
276
+ sumt += t; sumt2 += t*t; maxt = std::max(maxt, t);
277
+
278
+ // And now measure the time needed to quantize y and perform the dot product with the quantized y
279
+ t1 = std::chrono::high_resolution_clock::now();
280
+ float result;
281
+ if (scalar) {
282
+ quantize_row_q8_0_reference(y1.data(), q8.data(), kVecSize);
283
+ dot_q4_q8(kVecSize, &result, q40.data(), q8.data());
284
+ }
285
+ else {
286
+ const auto * vdot = ggml_get_type_traits_cpu(funcs_cpu->vec_dot_type);
287
+ vdot->from_float(y1.data(), q8.data(), kVecSize);
288
+ if (useQ4_1) funcs_cpu->vec_dot(kVecSize, &result, 0, q41.data(), 0, q8.data(), 0, 1);
289
+ else funcs_cpu->vec_dot(kVecSize, &result, 0, q40.data(), 0, q8.data(), 0, 1);
290
+ }
291
+ sumq += result;
292
+ t2 = std::chrono::high_resolution_clock::now();
293
+ t = 1e-3*std::chrono::duration_cast<std::chrono::nanoseconds>(t2-t1).count();
294
+ sumqt += t; sumqt2 += t*t; maxqt = std::max(maxqt, t);
295
+
296
+ }
297
+
298
+ // Report the time (and the average of the dot products so the compiler does not come up with the idea
299
+ // of optimizing away the function calls after figuring that the result is not used).
300
+ sum /= nloop; sumq /= nloop;
301
+ exactSum /= nloop;
302
+ printf("Exact result: <dot> = %g\n",exactSum);
303
+ printf("<dot> = %g, %g\n",sum,sumq);
304
+ sumt /= nloop; sumt2 /= nloop; sumt2 -= sumt*sumt;
305
+ if (sumt2 > 0) sumt2 = sqrt(sumt2);
306
+ printf("time = %g +/- %g us. maxt = %g us\n",sumt,sumt2,maxt);
307
+ sumqt /= nloop; sumqt2 /= nloop; sumqt2 -= sumqt*sumqt;
308
+ if (sumqt2 > 0) sumqt2 = sqrt(sumqt2);
309
+ printf("timeq = %g +/- %g us. maxt = %g us\n",sumqt,sumqt2,maxqt);
310
+ return 0;
311
+ }
pyproject.toml ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [project]
2
+ name = "llama-cpp-scripts"
3
+ description = "Scripts that ship with llama.cpp"
4
+ keywords = ["ggml", "gguf", "llama.cpp"]
5
+ version = "0.0.0"
6
+ dynamic = ["classifiers"]
7
+ readme = "README.md"
8
+ authors = [{name = "GGML", email = "ggml@ggml.ai"}]
9
+ requires-python = '>=3.10,<3.15'
10
+ dependencies = [
11
+ 'numpy (>=1.26.4,<3.0.0)',
12
+ 'sentencepiece (>=0.1.98,<0.3.0)',
13
+ 'transformers (==4.57.6)',
14
+ 'protobuf (>=4.21.0,<5.0.0)',
15
+ 'torch (>=2.6.0,<3.0.0)',
16
+ 'gguf @ ./gguf-py',
17
+ ]
18
+ classifiers = [
19
+ "Programming Language :: Python :: 3",
20
+ "License :: OSI Approved :: MIT License",
21
+ "Operating System :: OS Independent",
22
+ ]
23
+
24
+ [project.urls]
25
+ homepage = "https://ggml.ai"
26
+ repository = "https://github.com/ggml-org/llama.cpp"
27
+
28
+ [project.scripts]
29
+ llama-convert-hf-to-gguf = "convert_hf_to_gguf:main"
30
+ llama-convert-lora-to-gguf = "convert_lora_to_gguf:main"
31
+ llama-convert-llama-ggml-to-gguf = "convert_llama_ggml_to_gguf:main"
32
+ llama-ggml-vk-generate-shaders = "ggml_vk_generate_shaders:main"
33
+
34
+ [tool.poetry]
35
+ packages = [
36
+ { include = "*.py", from = "." },
37
+ { include = "conversion", from = "." },
38
+ ]
39
+
40
+ [tool.poetry.dependencies]
41
+ torch = [
42
+ { version = "==2.11.0", source = "pypi", markers = "sys_platform == 'darwin'" },
43
+ { version = "==2.11.0+cpu", source = "pytorch", markers = "sys_platform == 'linux'" },
44
+ { version = "==2.11.0", source = "pypi", markers = "sys_platform == 'win32'" },
45
+ ]
46
+
47
+ [tool.poetry.group.dev.dependencies]
48
+ pytest = "~=8.3.3"
49
+
50
+ # Force wheel + cpu
51
+ # For discussion and context see https://github.com/python-poetry/poetry#6409
52
+ [[tool.poetry.source]]
53
+ name = "pytorch"
54
+ url = "https://download.pytorch.org/whl/cpu"
55
+ priority = "explicit"
56
+
57
+ [tool.uv.sources]
58
+ torch = { index = "pytorch" }
59
+
60
+ [[tool.uv.index]]
61
+ name = "pytorch"
62
+ url = "https://download.pytorch.org/whl/cpu"
63
+ explicit = true
64
+
65
+ [build-system]
66
+ requires = ["poetry-core>=1.0.0"]
67
+ build-backend = "poetry.core.masonry.api"
pyrightconfig.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "extraPaths": ["gguf-py", "examples/model-conversion/scripts", "examples/model-conversion/scripts/utils"],
3
+ "pythonVersion": "3.9",
4
+ "pythonPlatform": "All",
5
+ "reportUnusedImport": "warning",
6
+ "reportDuplicateImport": "error",
7
+ "reportDeprecated": "warning",
8
+ "reportUnnecessaryTypeIgnoreComment": "information",
9
+ "disableBytesTypePromotions": false, // TODO: change once Python 3.12 is the minimum
10
+ "executionEnvironments": [
11
+ {
12
+ // TODO: make this version override work correctly
13
+ "root": "gguf-py",
14
+ "pythonVersion": "3.8",
15
+ },
16
+ {
17
+ // uses match expressions in steps.py
18
+ "root": "tools/server/tests",
19
+ "pythonVersion": "3.10",
20
+ },
21
+ ],
22
+ }
requirements.txt ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # These requirements include all dependencies for all top-level python scripts
2
+ # for llama.cpp. Avoid adding packages here directly.
3
+ #
4
+ # Package versions must stay compatible across all top-level python scripts.
5
+ #
6
+
7
+ -r ./requirements/requirements-convert_legacy_llama.txt
8
+
9
+ -r ./requirements/requirements-convert_hf_to_gguf.txt
10
+ -r ./requirements/requirements-convert_hf_to_gguf_update.txt
11
+ -r ./requirements/requirements-convert_llama_ggml_to_gguf.txt
12
+ -r ./requirements/requirements-convert_lora_to_gguf.txt
13
+ -r ./requirements/requirements-tool_bench.txt
requirements/requirements-all.txt ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ -r ../tools/mtmd/requirements.txt
2
+ -r ../tools/server/bench/requirements.txt
3
+ -r ../tools/server/tests/requirements.txt
4
+
5
+ -r ./requirements-compare-llama-bench.txt
6
+ -r ./requirements-server-bench.txt
7
+ -r ./requirements-pydantic.txt
8
+ -r ./requirements-test-tokenizer-random.txt
9
+
10
+ -r ./requirements-convert_hf_to_gguf.txt
11
+ -r ./requirements-convert_hf_to_gguf_update.txt
12
+ -r ./requirements-convert_legacy_llama.txt
13
+ -r ./requirements-convert_llama_ggml_to_gguf.txt
14
+ -r ./requirements-tool_bench.txt
15
+
16
+ -r ./requirements-gguf_editor_gui.txt
17
+
18
+ -r ../examples/model-conversion/requirements.txt
requirements/requirements-compare-llama-bench.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ tabulate~=0.9.0
2
+ GitPython~=3.1.43
3
+ matplotlib~=3.10.0
requirements/requirements-convert_hf_to_gguf.txt ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ -r ./requirements-convert_legacy_llama.txt
2
+ --extra-index-url https://download.pytorch.org/whl/cpu
3
+
4
+ ## Embedding Gemma requires PyTorch 2.6.0 or later, bumped to 2.11.0 for compatibility
5
+ torch==2.11.0
requirements/requirements-convert_hf_to_gguf_update.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ -r ./requirements-convert_legacy_llama.txt
requirements/requirements-convert_legacy_llama.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ numpy~=1.26.4
2
+ sentencepiece>=0.1.98,<0.3.0
3
+
4
+ transformers==4.57.6
5
+
6
+ gguf>=0.1.0
7
+ protobuf>=4.21.0,<5.0.0
requirements/requirements-convert_llama_ggml_to_gguf.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ -r ./requirements-convert_legacy_llama.txt
requirements/requirements-convert_lora_to_gguf.txt ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ -r ./requirements-convert_hf_to_gguf.txt
2
+ --extra-index-url https://download.pytorch.org/whl/cpu