Instructions to use HelloSun/lingtiny30vq3 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- llama.cpp
How to use HelloSun/lingtiny30vq3 with llama.cpp:
Install (macOS, Linux)
curl -LsSf https://llama.app/install.sh | sh # Start a local OpenAI-compatible server with a web UI: llama serve -hf HelloSun/lingtiny30vq3 # Run inference directly in the terminal: llama cli -hf HelloSun/lingtiny30vq3
Install from WinGet (Windows)
winget install llama.cpp # Start a local OpenAI-compatible server with a web UI: llama serve -hf HelloSun/lingtiny30vq3 # Run inference directly in the terminal: llama cli -hf HelloSun/lingtiny30vq3
Build from source code
git clone https://github.com/ggml-org/llama.cpp.git cd llama.cpp cmake -B build cmake --build build -j --target llama-app # Start a local OpenAI-compatible server with a web UI: ./build/bin/llama serve -hf HelloSun/lingtiny30vq3 # Run inference directly in the terminal: ./build/bin/llama cli -hf HelloSun/lingtiny30vq3
Use Docker
docker model run hf.co/HelloSun/lingtiny30vq3
- Llama new
- LM Studio
- Jan
- Ollama
How to use HelloSun/lingtiny30vq3 with Ollama:
ollama run hf.co/HelloSun/lingtiny30vq3
- Unsloth Desktop
- Pi
How to use HelloSun/lingtiny30vq3 with Pi:
Start the llama.cpp server
# Install llama.cpp: brew install llama.cpp # Start a local OpenAI-compatible server: llama serve -hf HelloSun/lingtiny30vq3
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "llama-cpp": { "baseUrl": "http://localhost:9931/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "HelloSun/lingtiny30vq3" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Docker Model Runner
How to use HelloSun/lingtiny30vq3 with Docker Model Runner:
docker model run hf.co/HelloSun/lingtiny30vq3
- Lemonade
How to use HelloSun/lingtiny30vq3 with Lemonade:
Pull the model
# Download Lemonade from https://lemonade-server.ai/ lemonade pull HelloSun/lingtiny30vq3
Run and chat with the model
lemonade run user.lingtiny30vq3-{{QUANT_TAG}}List all available models
lemonade list
- Hermes Agent
How to use HelloSun/lingtiny30vq3 with Hermes Agent:
Start the llama.cpp server
# Install llama.cpp: brew install llama.cpp # Start a local OpenAI-compatible server: llama serve -hf HelloSun/lingtiny30vq3
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:9931/v1 hermes config set model.default HelloSun/lingtiny30vq3
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use HelloSun/lingtiny30vq3 with OpenClaw:
Start the llama.cpp server
# Install llama.cpp: brew install llama.cpp # Start a local OpenAI-compatible server: llama serve -hf HelloSun/lingtiny30vq3
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:9931/v1 \ --custom-model-id "HelloSun/lingtiny30vq3" \ --custom-provider-id llama-cpp \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
LUT-VQ3-64: machine-tuned (threads/NUMA/ubatch), full 128k ctx data, one-click build, results
Browse files- README.md +113 -0
- build_and_run.sh +134 -0
- patches/0001-lut-vq3-64-avx512-vbmi-port-to-61dbf492.patch +922 -0
- results-128k/rss128k_base_q8dq6.txt +26 -0
- results-128k/rss128k_bf16.txt +26 -0
- results-128k/rss128k_q4_k_m.txt +26 -0
- results-128k/rss128k_q8_0.txt +26 -0
- results-128k/rss128k_vq3.txt +26 -0
- results-128k/rss128k_vq3_ub1024.txt +26 -0
- results-128k/rss128k_vq3_ub2048.txt +26 -0
- results-128k/run_base_q8dq6.txt +43 -0
- results-128k/run_bf16.txt +41 -0
- results-128k/run_q4_k_m.txt +43 -0
- results-128k/run_q8_0.txt +41 -0
- results-128k/run_vq3.txt +39 -0
- results-128k/run_vq3_ub1024.txt +42 -0
- results-128k/run_vq3_ub2048.txt +42 -0
- results-128k/tuning.tsv +26 -0
- results/bench.tsv +14 -0
- results/build_vq3.log +49 -0
- results/convert_std.log +733 -0
- results/gen_bf16_1.txt +52 -0
- results/gen_bf16_2.txt +41 -0
- results/gen_bf16_3.txt +47 -0
- results/gen_vq3_1.txt +46 -0
- results/gen_vq3_2.txt +45 -0
- results/gen_vq3_3.txt +47 -0
- results/kld_base_q8dq6.log +180 -0
- results/kld_q4_k_m.log +180 -0
- results/kld_q8_0.log +180 -0
- results/kld_vq3.log +180 -0
- results/ppl_base_q8dq6.log +14 -0
- results/ppl_bf16.log +14 -0
- results/ppl_q4_k_m.log +14 -0
- results/ppl_q8_0.log +14 -0
- results/ppl_std_hfconv.log +14 -0
- results/ppl_std_official.log +14 -0
- results/ppl_vq3.log +14 -0
- results/rss_base_q8dq6.txt +26 -0
- results/rss_bf16.txt +26 -0
- results/rss_q4_k_m.txt +26 -0
- results/rss_q8_0.txt +26 -0
- results/rss_vq3.txt +26 -0
- scripts/build_ling_vq3.py +256 -0
- scripts/run_experiments.sh +63 -0
- tests/test_vq3_port.c +108 -0
- vq3/vq3_codebook.h +231 -0
- vq3/vq3_enc_lib.c +369 -0
README.md
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: mit
|
| 3 |
+
base_model: inclusionAI/Ling-3.0-tiny
|
| 4 |
+
tags:
|
| 5 |
+
- llama.cpp
|
| 6 |
+
- quantization
|
| 7 |
+
- vector-quantization
|
| 8 |
+
- vq3
|
| 9 |
+
- moe
|
| 10 |
+
- gguf
|
| 11 |
+
library_name: gguf
|
| 12 |
+
---
|
| 13 |
+
|
| 14 |
+
# Ling-3.0-tiny LUT-VQ3-64(VQ3_0,專家張量 3.00 bpw)
|
| 15 |
+
|
| 16 |
+
`inclusionAI/Ling-3.0-tiny`(7.89B MoE,BF16 master,MIT)之 **LUT-VQ3-64 向量量化**版本。
|
| 17 |
+
46 個專家張量(`ffn_gate_exps`/`ffn_up_exps`)以 128 條目 3D int8 碼本 + 3-bit 子區塊尺度
|
| 18 |
+
量化(**3.00 bpw**),推論端為 llama.cpp 補丁新增的 `GGML_TYPE_VQ3_0` 核心
|
| 19 |
+
(AVX-512 VBMI `VPERMI2B` LUT 快速路徑)。**本 repo 版本針對 2×40C Xeon(雙 NUMA、
|
| 20 |
+
完整 AVX-512+AMX)調校**,含 128K 上下文實測數據。
|
| 21 |
+
|
| 22 |
+
- 模型:`Ling-3.0-tiny-VQ3.gguf`(4.66 GB / 4.34 GiB,n_tensors=526,有效率 4.73 BPW)
|
| 23 |
+
- 配方:gate/up_exps → VQ3_0;down_exps → Q6_K;其餘 → Q8_0
|
| 24 |
+
|
| 25 |
+
## 一鍵編譯 + 下載 + 實驗
|
| 26 |
+
|
| 27 |
+
```bash
|
| 28 |
+
git clone https://huggingface.co/HelloSun/lingtimy30vq3 && cd lingtimy30vq3
|
| 29 |
+
bash build_and_run.sh --all # 編譯→測試→下載模型→PPL/bench→128k 實驗
|
| 30 |
+
```
|
| 31 |
+
|
| 32 |
+
## 本機最佳化設定(2×40C Xeon 實測)
|
| 33 |
+
|
| 34 |
+
| 設定 | pp512 | tg128 | 結論 |
|
| 35 |
+
|---|---:|---:|---|
|
| 36 |
+
| `-t 32` | 268.9 | **38.0** | **decode 最快**(比 t64 快 +15%) |
|
| 37 |
+
| `-t 48` | 344.5 | 35.2 | 均衡 |
|
| 38 |
+
| `-t 64` | **353.3** | 33.8 | **prefill 最快** |
|
| 39 |
+
| `-t 80` | 68.3 | ~0 | ❌ 跨 NUMA 抖動,避免 |
|
| 40 |
+
| `--numa distribute` | 全線下降 | t80 崩至 0.9 | ❌ **本機不要用** |
|
| 41 |
+
|
| 42 |
+
**推薦:`-t 32 -tb 64`**(decode 32 執行緒 / prefill-batch 64 執行緒,兩者兼得)。
|
| 43 |
+
原始數據:[results/tuning.tsv](./results/tuning.tsv)。
|
| 44 |
+
|
| 45 |
+
## 短上下文基準(ctx 512–4096)
|
| 46 |
+
|
| 47 |
+
| 模型 | 大小 | PPL ↓ | KLD vs BF16 ↓ | pp512 | tg128 | 尖峰 RSS |
|
| 48 |
+
|---|---:|---:|---:|---:|---:|---:|
|
| 49 |
+
| BF16(master) | 14.71 GiB | **13.1045** | — | 425 t/s | 26.9 t/s | 15.70 GiB |
|
| 50 |
+
| Q8_0 | 7.82 GiB | 13.0718 | 0.012 | **504 t/s** | 33.0 t/s | 9.32 GiB |
|
| 51 |
+
| base-Q8dQ6 | 7.30 GiB | 13.1406 | 0.015 | 458 t/s | **34.2 t/s** | 8.80 GiB |
|
| 52 |
+
| Q4_K_M | 4.49 GiB | 13.4217 | 0.133 | 424 t/s | 33.7 t/s | 8.74 GiB |
|
| 53 |
+
| **VQ3(本 repo)** | **4.34 GiB** | 13.8465 | 0.255 | 369 t/s | 32.8 t/s | **5.84 GiB** |
|
| 54 |
+
|
| 55 |
+
明細:[PPL](./results/ppl_vq3.log) · [KLD](./results/kld_vq3.log) · [bench](./results/bench.tsv)
|
| 56 |
+
|
| 57 |
+
## ctx 128K 實測(`-c 131072`,prompt ≈131k tokens,`-t 32 -tb 64`)
|
| 58 |
+
|
| 59 |
+
| 模型 | ub | prefill(131k tokens) | decode(滿 128k KV) | 尖峰 RSS |
|
| 60 |
+
|---|---|---:|---:|---:|
|
| 61 |
+
| **VQ3** | 512 | 40.7 t/s | **6.5 t/s** | 6.10 GiB |
|
| 62 |
+
| **VQ3(最佳化)** | **1024** | **71.2 t/s(+75%)** | 6.0 t/s | 6.27 GiB |
|
| 63 |
+
| VQ3 | 2048 | 71.8 t/s | 6.1 t/s | 6.62 GiB |
|
| 64 |
+
| Q8_0 | 512 | 41.9 t/s | 6.2 t/s | 9.59 GiB |
|
| 65 |
+
| Q4_K_M | 512 | 41.4 t/s | 6.5 t/s | 9.18 GiB |
|
| 66 |
+
| base-Q8dQ6 | 512 | 41.6 t/s | 6.0 t/s | 9.06 GiB |
|
| 67 |
+
| BF16 | 512 | 39.7 t/s | 5.8 t/s | 15.95 GiB |
|
| 68 |
+
|
| 69 |
+
- **128k prefill 主要受 ubatch 影響**:`-ub 1024` 把 VQ3 prefill 從 40.7 提到 **71.2 t/s**
|
| 70 |
+
(ub2048 無進一步增益且 RSS +350 MiB → **ub 1024 為最佳點**)
|
| 71 |
+
- 128k decode 受 **KV 讀取**支配:MLA 壓縮 KV 每 token 576×2 B ×6 層,滿 context
|
| 72 |
+
每解碼一 token 需讀 ~0.9 GiB KV + 權重 → 頻寬受限;此時 VQ3 權重讀取量最小,
|
| 73 |
+
**decode 反超 Q8_0**(6.5 vs 6.2 t/s)
|
| 74 |
+
- 明細:[run_vq3.txt](./results-128k/run_vq3.txt) · [ub1024](./results-128k/run_vq3_ub1024.txt) · [各模型 RSS](./results-128k/)
|
| 75 |
+
|
| 76 |
+
## 使用
|
| 77 |
+
|
| 78 |
+
```bash
|
| 79 |
+
llama-cli -m Ling-3.0-tiny-VQ3.gguf -p "你好" -n 256 -t 32 -tb 64 -ngl 0
|
| 80 |
+
llama-server -m Ling-3.0-tiny-VQ3.gguf -t 32 -tb 64 -ngl 0 --host 0.0.0.0 --port 8080
|
| 81 |
+
# 128k 長文(本機最佳化:ub 1024)
|
| 82 |
+
llama-cli -m Ling-3.0-tiny-VQ3.gguf -f long.txt -c 131072 -n 256 -t 32 -tb 64 -ub 1024 -ngl 0
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
## 重新量化(可重現)
|
| 86 |
+
|
| 87 |
+
```bash
|
| 88 |
+
llama-quantize --tensor-type "ffn_down_exps=Q6_K" \
|
| 89 |
+
Ling-3.0-tiny-bf16.gguf Ling-3.0-tiny-base-Q8dQ6.gguf Q8_0
|
| 90 |
+
gcc -O3 -march=native -mavx512f -mf16c -fopenmp -shared -fPIC \
|
| 91 |
+
vq3/vq3_enc_lib.c -lm -o vq3/libvq3_enc.so
|
| 92 |
+
python3 scripts/build_ling_vq3.py \
|
| 93 |
+
--base-gguf Ling-3.0-tiny-base-Q8dQ6.gguf \
|
| 94 |
+
--bf16-gguf Ling-3.0-tiny-bf16.gguf --output Ling-3.0-tiny-VQ3.gguf
|
| 95 |
+
```
|
| 96 |
+
|
| 97 |
+
## 連結
|
| 98 |
+
|
| 99 |
+
- 基礎模型:[inclusionAI/Ling-3.0-tiny](https://huggingface.co/inclusionAI/Ling-3.0-tiny)(MIT)
|
| 100 |
+
- 上游 llama.cpp:[ggml-org/llama.cpp @ `61dbf492`](https://github.com/ggml-org/llama.cpp/tree/61dbf49228c8a96b72e40c89d93cb7c003411f8c)
|
| 101 |
+
- VQ3 格式:[synchopate/llama-modchip](https://github.com/synchopate/llama-modchip)(LUT-VQ3-64)
|
| 102 |
+
- 舊版 repo(vp3 拼字):[HelloSun/lingtimy30vp3](https://huggingface.co/HelloSun/lingtimy30vp3)
|
| 103 |
+
|
| 104 |
+
## 注意事項
|
| 105 |
+
|
| 106 |
+
- 補丁含 **AMX 修復**:3D MLA 權重(`attn_k_b`/`attn_v_b`)被 AMX buffer 認領後,
|
| 107 |
+
bailingmoe3 以 permuted view 之 MUL_MAT 消費會崩潰;已於 `ggml-cpu/amx/amx.cpp`
|
| 108 |
+
加 `ggml_n_dims(src0)==2` 檢查(BF16/Q8_0/Q4_K_M 同樣需要)。
|
| 109 |
+
- 核心正確性:SQNR 13.45 dB、cos 0.977 @ 3.00 bpw(`tests/test_vq3_port.c`)。
|
| 110 |
+
|
| 111 |
+
## 授權
|
| 112 |
+
|
| 113 |
+
MIT(同基礎模型)。
|
build_and_run.sh
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# =============================================================================
|
| 3 |
+
# build_and_run.sh — LUT-VQ3-64 for Ling-3.0-tiny:一鍵編譯 + 驗證 + 下載 + 實驗
|
| 4 |
+
#
|
| 5 |
+
# bash build_and_run.sh # 編譯 + 單元測試
|
| 6 |
+
# bash build_and_run.sh --download # 另外下載 VQ3 GGUF 並做推論冒煙測試
|
| 7 |
+
# bash build_and_run.sh --experiments # 另外跑 PPL + llama-bench + 生成樣本
|
| 8 |
+
# bash build_and_run.sh --ctx128k # 另外跑 128k 上下文實驗(prefill/decode/RSS)
|
| 9 |
+
# bash build_and_run.sh --all # 全部
|
| 10 |
+
#
|
| 11 |
+
# 需求:git, cmake>=3.14, gcc/g++(x86-64 建議 AVX-512;VQ3 LUT 核心需 VBMI),
|
| 12 |
+
# python3(僅重量化時需要)
|
| 13 |
+
# 模型:inclusionAI/Ling-3.0-tiny(MIT)之 LUT-VQ3-64 量化(專家張量 3.00 bpw)
|
| 14 |
+
#
|
| 15 |
+
# ※ 本機最佳化(2×40C Xeon,雙 NUMA)實測結論:
|
| 16 |
+
# - decode(tg)用 -t 32 最快(38.9 t/s);prefill(pp)用 -t 64 最快(353+ t/s)
|
| 17 |
+
# - 故預設採用 -t 32 -tb 64(decode 32 執行緒 / prefill 64 執行緒)
|
| 18 |
+
# - 【不要】加 --numa distribute:實測 tg@t80 崩至 0.9 t/s、pp@t48 −21%
|
| 19 |
+
# - 80 執行緒會跨 NUMA 抖動(pp 崩至 ~70 t/s),避免使用
|
| 20 |
+
# =============================================================================
|
| 21 |
+
set -euo pipefail
|
| 22 |
+
|
| 23 |
+
LLAMA_REPO="https://github.com/ggml-org/llama.cpp"
|
| 24 |
+
LLAMA_COMMIT="61dbf49228c8a96b72e40c89d93cb7c003411f8c"
|
| 25 |
+
HF_REPO="HelloSun/lingtiny30vq3"
|
| 26 |
+
MODEL_NAME="Ling-3.0-tiny-VQ3.gguf"
|
| 27 |
+
MODEL_URL="https://huggingface.co/${HF_REPO}/resolve/main/${MODEL_NAME}"
|
| 28 |
+
|
| 29 |
+
HERE="$(cd "$(dirname "$0")" && pwd)"
|
| 30 |
+
WORK="${HERE}/work"
|
| 31 |
+
JOBS="$(nproc)"
|
| 32 |
+
T_DEC=32 # decode 最佳執行緒數(本機實測)
|
| 33 |
+
T_BATCH=64 # prefill/batch 最佳執行緒數(本機實測)
|
| 34 |
+
DO_DOWNLOAD=0; DO_EXPERIMENTS=0; DO_CTX128K=0
|
| 35 |
+
while [[ $# -gt 0 ]]; do
|
| 36 |
+
case "$1" in
|
| 37 |
+
--jobs) JOBS="$2"; shift 2;;
|
| 38 |
+
--download) DO_DOWNLOAD=1; shift;;
|
| 39 |
+
--experiments) DO_EXPERIMENTS=1; shift;;
|
| 40 |
+
--ctx128k) DO_CTX128K=1; shift;;
|
| 41 |
+
--all) DO_DOWNLOAD=1; DO_EXPERIMENTS=1; DO_CTX128K=1; shift;;
|
| 42 |
+
*) echo "unknown arg: $1"; exit 1;;
|
| 43 |
+
esac
|
| 44 |
+
done
|
| 45 |
+
|
| 46 |
+
echo "== [1/6] 取得 llama.cpp @ ${LLAMA_COMMIT:0:9} =="
|
| 47 |
+
mkdir -p "${WORK}"
|
| 48 |
+
if [[ ! -d "${WORK}/llama.cpp" ]]; then
|
| 49 |
+
git clone "${LLAMA_REPO}" "${WORK}/llama.cpp"
|
| 50 |
+
fi
|
| 51 |
+
git -C "${WORK}/llama.cpp" fetch --depth 1 origin "${LLAMA_COMMIT}" 2>/dev/null || true
|
| 52 |
+
git -C "${WORK}/llama.cpp" checkout -q --detach "${LLAMA_COMMIT}"
|
| 53 |
+
|
| 54 |
+
echo "== [2/6] 套用 LUT-VQ3-64 補丁 =="
|
| 55 |
+
if git -C "${WORK}/llama.cpp" apply --check "${HERE}/patches/0001-lut-vq3-64-avx512-vbmi-port-to-61dbf492.patch" 2>/dev/null; then
|
| 56 |
+
git -C "${WORK}/llama.cpp" apply "${HERE}/patches/0001-lut-vq3-64-avx512-vbmi-port-to-61dbf492.patch"
|
| 57 |
+
echo "patch applied"
|
| 58 |
+
elif grep -rq "GGML_TYPE_VQ3_0" "${WORK}/llama.cpp/ggml/include/ggml.h"; then
|
| 59 |
+
echo "patch already applied"
|
| 60 |
+
else
|
| 61 |
+
echo "ERROR: patch does not apply"; exit 1
|
| 62 |
+
fi
|
| 63 |
+
|
| 64 |
+
echo "== [3/6] 編譯(-j ${JOBS},GGML_NATIVE=ON:AVX-512/AMX 自動啟用)=="
|
| 65 |
+
cmake -S "${WORK}/llama.cpp" -B "${WORK}/llama.cpp/build" \
|
| 66 |
+
-DGGML_NATIVE=ON -DLLAMA_CURL=OFF -DBUILD_SHARED_LIBS=OFF \
|
| 67 |
+
-DCMAKE_BUILD_TYPE=Release
|
| 68 |
+
cmake --build "${WORK}/llama.cpp/build" -j "${JOBS}" \
|
| 69 |
+
--target llama-cli llama-server llama-quantize llama-perplexity llama-bench
|
| 70 |
+
BIN="${WORK}/llama.cpp/build/bin"
|
| 71 |
+
|
| 72 |
+
echo "== [4/6] 編譯 VQ3 編碼器函式庫(量化用)=="
|
| 73 |
+
mkdir -p "${WORK}/vq3"
|
| 74 |
+
cp "${HERE}/vq3/vq3_enc_lib.c" "${HERE}/vq3/vq3_codebook.h" "${WORK}/vq3/"
|
| 75 |
+
gcc -O3 -march=native -mavx512f -mf16c -fopenmp -shared -fPIC \
|
| 76 |
+
"${WORK}/vq3/vq3_enc_lib.c" -lm -o "${WORK}/vq3/libvq3_enc.so"
|
| 77 |
+
|
| 78 |
+
echo "== [5/6] 單元測試(kernel + 格式)=="
|
| 79 |
+
gcc -O2 -march=native -mavx512f -mf16c -lgomp \
|
| 80 |
+
-I"${WORK}/llama.cpp/ggml/include" -I"${WORK}/llama.cpp/ggml/src" \
|
| 81 |
+
"${HERE}/tests/test_vq3_port.c" \
|
| 82 |
+
"${WORK}/llama.cpp/build/ggml/src/libggml-cpu.a" \
|
| 83 |
+
"${WORK}/llama.cpp/build/ggml/src/libggml-base.a" \
|
| 84 |
+
"${WORK}/llama.cpp/build/ggml/src/libggml.a" \
|
| 85 |
+
-lm -lpthread -o "${WORK}/test_vq3_port" \
|
| 86 |
+
|| echo "note: 測試執行檔連結失敗可略過,不影響推論"
|
| 87 |
+
"${WORK}/test_vq3_port" 2>/dev/null || true
|
| 88 |
+
|
| 89 |
+
echo "== [6/6] 完成 =="
|
| 90 |
+
echo " 推論: ${BIN}/llama-cli -m <vq3.gguf> -p \"你好\" -n 256 -t ${T_DEC} -tb ${T_BATCH} -ngl 0"
|
| 91 |
+
echo " Server: ${BIN}/llama-server -m <vq3.gguf> -t ${T_DEC} -tb ${T_BATCH} -ngl 0 --host 0.0.0.0"
|
| 92 |
+
echo " 量化: python3 ${HERE}/scripts/build_ling_vq3.py --help"
|
| 93 |
+
|
| 94 |
+
if [[ "${DO_DOWNLOAD}" == "1" ]]; then
|
| 95 |
+
echo "== 下載 ${MODEL_NAME} =="
|
| 96 |
+
mkdir -p "${WORK}/models"
|
| 97 |
+
if command -v huggingface-cli >/dev/null 2>&1; then
|
| 98 |
+
huggingface-cli download "${HF_REPO}" "${MODEL_NAME}" --local-dir "${WORK}/models"
|
| 99 |
+
else
|
| 100 |
+
curl -L --fail -o "${WORK}/models/${MODEL_NAME}" "${MODEL_URL}"
|
| 101 |
+
fi
|
| 102 |
+
echo "== 推論冒煙測試 =="
|
| 103 |
+
"${BIN}/llama-cli" -m "${WORK}/models/${MODEL_NAME}" \
|
| 104 |
+
-p "你好,請用一句話介紹台灣。" -n 64 -t "${T_DEC}" -tb "${T_BATCH}" -ngl 0 --temp 0 -st
|
| 105 |
+
fi
|
| 106 |
+
|
| 107 |
+
if [[ "${DO_EXPERIMENTS}" == "1" ]]; then
|
| 108 |
+
MODEL="${WORK}/models/${MODEL_NAME}"
|
| 109 |
+
[[ -f "${MODEL}" ]] || { echo "需先 --download"; exit 1; }
|
| 110 |
+
echo "== PPL(wiki.test,256 chunks)=="
|
| 111 |
+
[[ -f "${WORK}/wiki.test.raw" ]] || { \
|
| 112 |
+
curl -sL -o "${WORK}/wt2.zip" "https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip" && \
|
| 113 |
+
unzip -o -j "${WORK}/wt2.zip" "wikitext-2-raw/wiki.test.raw" -d "${WORK}"; }
|
| 114 |
+
"${BIN}/llama-perplexity" -m "${MODEL}" -f "${WORK}/wiki.test.raw" \
|
| 115 |
+
-c 512 --chunks 256 -t "${T_BATCH}" -ngl 0
|
| 116 |
+
echo "== llama-bench(本機最佳執行緒)=="
|
| 117 |
+
"${BIN}/llama-bench" -t "${T_DEC},${T_BATCH}" -p 512 -n 128 -r 2 -m "${MODEL}" -ngl 0
|
| 118 |
+
fi
|
| 119 |
+
|
| 120 |
+
if [[ "${DO_CTX128K}" == "1" ]]; then
|
| 121 |
+
MODEL="${WORK}/models/${MODEL_NAME}"
|
| 122 |
+
[[ -f "${MODEL}" ]] || { echo "需先 --download"; exit 1; }
|
| 123 |
+
R="${WORK}/ctx128k"; mkdir -p "${R}"
|
| 124 |
+
echo "== 建立 ~120k token prompt =="
|
| 125 |
+
[[ -f "${R}/prompt_120k.txt" ]] || { \
|
| 126 |
+
curl -sL -o "${R}/wt2train.zip" "https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip" && \
|
| 127 |
+
unzip -o -j "${R}/wt2train.zip" "wikitext-2-raw/wiki.train.raw" -d "${R}" && \
|
| 128 |
+
head -c 520000 "${R}/wiki.train.raw" > "${R}/prompt_120k.txt"; }
|
| 129 |
+
echo "== 128k 上下文:prefill ~120k tokens + decode 64(-t ${T_DEC} -tb ${T_BATCH})=="
|
| 130 |
+
/usr/bin/time -v "${BIN}/llama-cli" -m "${MODEL}" -f "${R}/prompt_120k.txt" \
|
| 131 |
+
-c 131072 -n 64 -t "${T_DEC}" -tb "${T_BATCH}" -ub 1024 -ngl 0 --no-warmup --temp 0 -st \
|
| 132 |
+
2> >(tee "${R}/rss_128k.txt" >&2) | grep -E "prompt eval time|eval time|Maximum resident"
|
| 133 |
+
echo "(rss_128k.txt 內含尖峰駐留記憶體)"
|
| 134 |
+
fi
|
patches/0001-lut-vq3-64-avx512-vbmi-port-to-61dbf492.patch
ADDED
|
@@ -0,0 +1,922 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/ggml/src/vq3_codebook.h b/ggml/src/vq3_codebook.h
|
| 2 |
+
new file mode 100644
|
| 3 |
+
index 0000000..ee684a0
|
| 4 |
+
--- /dev/null
|
| 5 |
+
+++ b/ggml/src/vq3_codebook.h
|
| 6 |
+
@@ -0,0 +1,231 @@
|
| 7 |
+
+#ifndef GGML_VQ3_CODEBOOK_H
|
| 8 |
+
+#define GGML_VQ3_CODEBOOK_H
|
| 9 |
+
+
|
| 10 |
+
+#include <stdint.h>
|
| 11 |
+
+
|
| 12 |
+
+#define GGML_VQ3_G (8.6373016238e-03f)
|
| 13 |
+
+
|
| 14 |
+
+#if defined(_MSC_VER)
|
| 15 |
+
+#define GGML_VQ3_ALIGN64 __declspec(align(64))
|
| 16 |
+
+#else
|
| 17 |
+
+#define GGML_VQ3_ALIGN64 __attribute__((aligned(64)))
|
| 18 |
+
+#endif
|
| 19 |
+
+
|
| 20 |
+
+static const int8_t ggml_vq3_cb[128][3] = {
|
| 21 |
+
+ { -24, -19, -24 },
|
| 22 |
+
+ { -37, -20, -123 },
|
| 23 |
+
+ { -41, -18, 18 },
|
| 24 |
+
+ { 23, -6, 27 },
|
| 25 |
+
+ { 29, -15, -65 },
|
| 26 |
+
+ { -3, 4, 0 },
|
| 27 |
+
+ { -59, -25, 72 },
|
| 28 |
+
+ { -68, -13, -66 },
|
| 29 |
+
+ { 11, -54, -122 },
|
| 30 |
+
+ { -84, -5, 30 },
|
| 31 |
+
+ { 35, -51, 51 },
|
| 32 |
+
+ { -19, 8, 50 },
|
| 33 |
+
+ { 54, 9, 62 },
|
| 34 |
+
+ { 12, -74, -55 },
|
| 35 |
+
+ { -41, -73, -34 },
|
| 36 |
+
+ { 16, 46, 19 },
|
| 37 |
+
+ { -62, -70, -111 },
|
| 38 |
+
+ { 69, -14, 116 },
|
| 39 |
+
+ { -65, -73, 20 },
|
| 40 |
+
+ { 70, 11, 12 },
|
| 41 |
+
+ { 75, -2, -40 },
|
| 42 |
+
+ { 55, 43, -65 },
|
| 43 |
+
+ { -6, 62, -121 },
|
| 44 |
+
+ { -47, 32, 16 },
|
| 45 |
+
+ { 73, -64, 65 },
|
| 46 |
+
+ { -22, 67, 121 },
|
| 47 |
+
+ { 62, 61, -9 },
|
| 48 |
+
+ { 5, 80, -18 },
|
| 49 |
+
+ { -123, -46, 55 },
|
| 50 |
+
+ { -119, -58, -56 },
|
| 51 |
+
+ { -24, -120, -122 },
|
| 52 |
+
+ { -57, 48, -72 },
|
| 53 |
+
+ { -125, -52, -3 },
|
| 54 |
+
+ { -30, 77, 27 },
|
| 55 |
+
+ { -19, -126, -11 },
|
| 56 |
+
+ { 43, -118, -118 },
|
| 57 |
+
+ { -56, 69, -22 },
|
| 58 |
+
+ { 52, -122, 119 },
|
| 59 |
+
+ { 120, 6, -116 },
|
| 60 |
+
+ { 123, -22, -59 },
|
| 61 |
+
+ { 124, -26, 115 },
|
| 62 |
+
+ { 127, 4, -10 },
|
| 63 |
+
+ { -97, -121, -34 },
|
| 64 |
+
+ { 119, 46, -48 },
|
| 65 |
+
+ { 117, 54, -115 },
|
| 66 |
+
+ { 7, 117, 76 },
|
| 67 |
+
+ { 9, 120, 26 },
|
| 68 |
+
+ { -122, -113, 102 },
|
| 69 |
+
+ { -123, -114, 19 },
|
| 70 |
+
+ { 5, 124, -58 },
|
| 71 |
+
+ { 51, 122, -8 },
|
| 72 |
+
+ { 100, -123, 51 },
|
| 73 |
+
+ { 114, -111, -62 },
|
| 74 |
+
+ { 62, 123, 52 },
|
| 75 |
+
+ { -56, 119, -59 },
|
| 76 |
+
+ { -122, 6, -31 },
|
| 77 |
+
+ { -123, 8, 120 },
|
| 78 |
+
+ { -121, 40, -121 },
|
| 79 |
+
+ { 122, -120, -3 },
|
| 80 |
+
+ { -127, 1, 15 },
|
| 81 |
+
+ { 114, 114, 111 },
|
| 82 |
+
+ { 121, 114, 28 },
|
| 83 |
+
+ { -114, 97, 118 },
|
| 84 |
+
+ { -119, 116, 48 },
|
| 85 |
+
+ { 5, -19, 75 },
|
| 86 |
+
+ { 16, -14, 126 },
|
| 87 |
+
+ { 10, 8, -122 },
|
| 88 |
+
+ { -18, -41, -70 },
|
| 89 |
+
+ { 18, -35, -16 },
|
| 90 |
+
+ { -13, -51, 18 },
|
| 91 |
+
+ { -37, -47, 123 },
|
| 92 |
+
+ { 29, 16, -25 },
|
| 93 |
+
+ { -15, 8, -63 },
|
| 94 |
+
+ { -15, 14, 120 },
|
| 95 |
+
+ { -19, -71, 69 },
|
| 96 |
+
+ { -75, -42, -18 },
|
| 97 |
+
+ { 31, -64, 121 },
|
| 98 |
+
+ { 60, -2, -118 },
|
| 99 |
+
+ { -15, 37, -26 },
|
| 100 |
+
+ { 1, 49, 66 },
|
| 101 |
+
+ { 20, -81, 0 },
|
| 102 |
+
+ { 43, 42, 120 },
|
| 103 |
+
+ { 66, -41, 8 },
|
| 104 |
+
+ { 63, -53, -98 },
|
| 105 |
+
+ { 4, 58, -67 },
|
| 106 |
+
+ { 63, -62, -38 },
|
| 107 |
+
+ { -115, -30, -124 },
|
| 108 |
+
+ { -108, -47, 118 },
|
| 109 |
+
+ { 53, 59, -124 },
|
| 110 |
+
+ { -54, 31, -125 },
|
| 111 |
+
+ { 58, 65, 56 },
|
| 112 |
+
+ { -62, 19, 123 },
|
| 113 |
+
+ { -5, -119, 115 },
|
| 114 |
+
+ { -66, 13, -23 },
|
| 115 |
+
+ { -24, -121, 38 },
|
| 116 |
+
+ { -31, -120, -66 },
|
| 117 |
+
+ { -61, -113, 118 },
|
| 118 |
+
+ { -63, 46, 65 },
|
| 119 |
+
+ { 30, -121, 48 },
|
| 120 |
+
+ { 35, -124, -54 },
|
| 121 |
+
+ { 118, -18, 34 },
|
| 122 |
+
+ { -76, -122, 46 },
|
| 123 |
+
+ { 56, -122, -4 },
|
| 124 |
+
+ { 118, 26, 62 },
|
| 125 |
+
+ { 122, -48, -117 },
|
| 126 |
+
+ { 115, 44, 120 },
|
| 127 |
+
+ { 122, -56, -12 },
|
| 128 |
+
+ { 38, 112, 124 },
|
| 129 |
+
+ { -113, -115, -109 },
|
| 130 |
+
+ { 124, -70, 44 },
|
| 131 |
+
+ { 121, 55, 14 },
|
| 132 |
+
+ { 63, 110, -59 },
|
| 133 |
+
+ { 25, 123, -120 },
|
| 134 |
+
+ { -47, 113, -121 },
|
| 135 |
+
+ { -24, 124, -9 },
|
| 136 |
+
+ { 116, -106, 119 },
|
| 137 |
+
+ { -51, 117, 60 },
|
| 138 |
+
+ { -40, 124, 122 },
|
| 139 |
+
+ { 114, -118, -124 },
|
| 140 |
+
+ { -115, 55, 13 },
|
| 141 |
+
+ { -124, 4, -76 },
|
| 142 |
+
+ { -124, 26, 64 },
|
| 143 |
+
+ { -74, 117, 8 },
|
| 144 |
+
+ { -119, 59, -46 },
|
| 145 |
+
+ { 110, 120, -115 },
|
| 146 |
+
+ { 121, 116, -38 },
|
| 147 |
+
+ { -115, 114, -106 },
|
| 148 |
+
+ { -122, 121, -25 },
|
| 149 |
+
+};
|
| 150 |
+
+
|
| 151 |
+
+GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut0_lo[64] = { -24, -37, -41, 23, 29, -3, -59, -68, 11, -84, 35, -19, 54, 12, -41, 16, -62, 69, -65, 70, 75, 55, -6, -47, 73, -22, 62, 5, -123, -119, -24, -57, -125, -30, -19, 43, -56, 52, 120, 123, 124, 127, -97, 119, 117, 7, 9, -122, -123, 5, 51, 100, 114, 62, -56, -122, -123, -121, 122, -127, 114, 121, -114, -119 };
|
| 152 |
+
+GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut0_hi[64] = { 5, 16, 10, -18, 18, -13, -37, 29, -15, -15, -19, -75, 31, 60, -15, 1, 20, 43, 66, 63, 4, 63, -115, -108, 53, -54, 58, -62, -5, -66, -24, -31, -61, -63, 30, 35, 118, -76, 56, 118, 122, 115, 122, 38, -113, 124, 121, 63, 25, -47, -24, 116, -51, -40, 114, -115, -124, -124, -74, -119, 110, 121, -115, -122 };
|
| 153 |
+
+GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut1_lo[64] = { -19, -20, -18, -6, -15, 4, -25, -13, -54, -5, -51, 8, 9, -74, -73, 46, -70, -14, -73, 11, -2, 43, 62, 32, -64, 67, 61, 80, -46, -58, -120, 48, -52, 77, -126, -118, 69, -122, 6, -22, -26, 4, -121, 46, 54, 117, 120, -113, -114, 124, 122, -123, -111, 123, 119, 6, 8, 40, -120, 1, 114, 114, 97, 116 };
|
| 154 |
+
+GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut1_hi[64] = { -19, -14, 8, -41, -35, -51, -47, 16, 8, 14, -71, -42, -64, -2, 37, 49, -81, 42, -41, -53, 58, -62, -30, -47, 59, 31, 65, 19, -119, 13, -121, -120, -113, 46, -121, -124, -18, -122, -122, 26, -48, 44, -56, 112, -115, -70, 55, 110, 123, 113, 124, -106, 117, 124, -118, 55, 4, 26, 117, 59, 120, 116, 114, 121 };
|
| 155 |
+
+GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut2_lo[64] = { -24, -123, 18, 27, -65, 0, 72, -66, -122, 30, 51, 50, 62, -55, -34, 19, -111, 116, 20, 12, -40, -65, -121, 16, 65, 121, -9, -18, 55, -56, -122, -72, -3, 27, -11, -118, -22, 119, -116, -59, 115, -10, -34, -48, -115, 76, 26, 102, 19, -58, -8, 51, -62, 52, -59, -31, 120, -121, -3, 15, 111, 28, 118, 48 };
|
| 156 |
+
+GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut2_hi[64] = { 75, 126, -122, -70, -16, 18, 123, -25, -63, 120, 69, -18, 121, -118, -26, 66, 0, 120, 8, -98, -67, -38, -124, 118, -124, -125, 56, 123, 115, -23, 38, -66, 118, 65, 48, -54, 34, 46, -4, 62, -117, 120, -12, 124, -109, 44, 14, -59, -120, -121, -9, 119, 60, 122, -124, 13, -76, 64, 8, -46, -115, -38, -106, -25 };
|
| 157 |
+
+
|
| 158 |
+
+GGML_VQ3_ALIGN64 static const uint8_t ggml_vq3_perm_rep64[64] = {
|
| 159 |
+
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,
|
| 160 |
+
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,
|
| 161 |
+
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19
|
| 162 |
+
+};
|
| 163 |
+
+
|
| 164 |
+
+GGML_VQ3_ALIGN64 static const uint8_t ggml_vq3_shuf_sc_gather[64] = {
|
| 165 |
+
+ 0, 1, 2,63, 3, 4, 5,63, 6, 7, 8,63, 9,10,11,63,
|
| 166 |
+
+ 12,13,14,63, 15,16,17,63, 18,19,20,63, 21,63,20,21,
|
| 167 |
+
+ 63,63,63,63, 63,63,63,63, 63,63,63,63, 63,63,63,63,
|
| 168 |
+
+ 63,63,63,63, 63,63,63,63, 63,63,63,63, 63,63,63,63
|
| 169 |
+
+};
|
| 170 |
+
+
|
| 171 |
+
+GGML_VQ3_ALIGN64 static const uint16_t ggml_vq3_sc_rshift[32] = {
|
| 172 |
+
+ 7,7, 7,7, 7,7, 7,7, 7,7, 7,7, 7,7, 7,6,
|
| 173 |
+
+ 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0
|
| 174 |
+
+};
|
| 175 |
+
+
|
| 176 |
+
+GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_weights_sc[64] = {
|
| 177 |
+
+ 1,2,4,0, 1,2,4,0, 1,2,4,0, 1,2,4,0,
|
| 178 |
+
+ 1,2,4,0, 1,2,4,0, 1,2,4,0, 1,0,2,4,
|
| 179 |
+
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
|
| 180 |
+
+ 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
|
| 181 |
+
+};
|
| 182 |
+
+
|
| 183 |
+
+GGML_VQ3_ALIGN64 static const uint16_t ggml_vq3_shuf_mult_i16[32] = {
|
| 184 |
+
+ 0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6,
|
| 185 |
+
+ 8, 8, 8, 8, 10,10,10,10, 12,12,12,12, 14,14,14,14
|
| 186 |
+
+};
|
| 187 |
+
+
|
| 188 |
+
+#if defined(__AVX512F__) || (defined(__BMI2__) && (defined(__x86_64__) || defined(_M_X64)))
|
| 189 |
+
+#include <immintrin.h>
|
| 190 |
+
+#endif
|
| 191 |
+
+#if defined(__AVX512F__)
|
| 192 |
+
+#if defined(__GNUC__) || defined(__clang__)
|
| 193 |
+
+static inline __m512i ggml_vq3_loadu_si512(const void * p) {
|
| 194 |
+
+ __m512i r;
|
| 195 |
+
+ __asm__("vmovdqu64 %1, %0" : "=v"(r) : "m"(*(const uint64_t *)p));
|
| 196 |
+
+ return r;
|
| 197 |
+
+}
|
| 198 |
+
+#else
|
| 199 |
+
+#define ggml_vq3_loadu_si512(p) _mm512_loadu_si512((const __m512i *)(p))
|
| 200 |
+
+#endif
|
| 201 |
+
+#endif
|
| 202 |
+
+
|
| 203 |
+
+static inline uint64_t ggml_vq3_unpack_scales(uint32_t s24) {
|
| 204 |
+
+#if defined(__BMI2__) && (defined(__x86_64__) || defined(_M_X64))
|
| 205 |
+
+ return _pdep_u64((uint64_t)s24, 0x0707070707070707ULL) + 0x0202020202020202ULL;
|
| 206 |
+
+#else
|
| 207 |
+
+ uint64_t m_all = 0x0202020202020202ULL;
|
| 208 |
+
+ for (int b = 0; b < 8; ++b) {
|
| 209 |
+
+ m_all += ((uint64_t)((s24 >> (3 * b)) & 0x7U)) << (8 * b);
|
| 210 |
+
+ }
|
| 211 |
+
+ return m_all;
|
| 212 |
+
+#endif
|
| 213 |
+
+}
|
| 214 |
+
+
|
| 215 |
+
+GGML_VQ3_ALIGN64 static const uint32_t ggml_vq3_shuf_mult512[16] = {
|
| 216 |
+
+ 0x80008000, 0x80008000, 0x80018001, 0x80018001,
|
| 217 |
+
+ 0x80028002, 0x80028002, 0x80038003, 0x80038003,
|
| 218 |
+
+ 0x80048004, 0x80048004, 0x80058005, 0x80058005,
|
| 219 |
+
+ 0x80068006, 0x80068006, 0x80078007, 0x80078007
|
| 220 |
+
+};
|
| 221 |
+
+
|
| 222 |
+
+GGML_VQ3_ALIGN64 static const uint32_t ggml_q2_lut_abs512[16] = {
|
| 223 |
+
+ 0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401,
|
| 224 |
+
+ 0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401,
|
| 225 |
+
+ 0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401,
|
| 226 |
+
+ 0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401
|
| 227 |
+
+};
|
| 228 |
+
+
|
| 229 |
+
+GGML_VQ3_ALIGN64 static const uint32_t ggml_q2_shift_qs[16] = {
|
| 230 |
+
+ 0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6
|
| 231 |
+
+};
|
| 232 |
+
+
|
| 233 |
+
+GGML_VQ3_ALIGN64 static const uint64_t ggml_q2_ror_qh[8] = {
|
| 234 |
+
+ 62, 63, 0, 1, 2, 3, 4, 5
|
| 235 |
+
+};
|
| 236 |
+
+
|
| 237 |
+
+#endif // GGML_VQ3_CODEBOOK_H
|
| 238 |
+
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
|
| 239 |
+
index 2341fa6..e64236b 100644
|
| 240 |
+
--- a/ggml/include/ggml.h
|
| 241 |
+
+++ b/ggml/include/ggml.h
|
| 242 |
+
@@ -260,7 +260,8 @@ extern "C" {
|
| 243 |
+
GGML_TYPE_Q1_0 = 41,
|
| 244 |
+
GGML_TYPE_Q2_0 = 42,
|
| 245 |
+
GGML_TYPE_F8_E4M3 = 43,
|
| 246 |
+
- GGML_TYPE_COUNT = 44,
|
| 247 |
+
+ GGML_TYPE_VQ3_0 = 44, // LUT-VQ3-64 (24 B / 64 W, 3.00 bpw) [llama-modchip port]
|
| 248 |
+
+ GGML_TYPE_COUNT = 45,
|
| 249 |
+
};
|
| 250 |
+
|
| 251 |
+
// [TAG_GGML_PREC]
|
| 252 |
+
@@ -316,6 +317,7 @@ extern "C" {
|
| 253 |
+
GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors
|
| 254 |
+
GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors
|
| 255 |
+
GGML_FTYPE_MOSTLY_F8_E4M3 = 29, // except 1d tensors
|
| 256 |
+
+ GGML_FTYPE_MOSTLY_VQ3_0 = 30, // except 1d tensors [llama-modchip port]
|
| 257 |
+
};
|
| 258 |
+
|
| 259 |
+
// available tensor operations:
|
| 260 |
+
diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h
|
| 261 |
+
index 1dbbe32..f7fb355 100644
|
| 262 |
+
--- a/ggml/src/ggml-common.h
|
| 263 |
+
+++ b/ggml/src/ggml-common.h
|
| 264 |
+
@@ -191,6 +191,15 @@ typedef struct {
|
| 265 |
+
} block_q2_0;
|
| 266 |
+
static_assert(sizeof(block_q2_0) == sizeof(ggml_half) + QK2_0 / 4, "wrong q2_0 block size/padding");
|
| 267 |
+
|
| 268 |
+
+#define QK_VQ3_0 64
|
| 269 |
+
+// LUT-VQ3-64 [llama-modchip port]: 2 B FP16 super-block scale + 22 B payload
|
| 270 |
+
+// (21 x 7-bit 3D codebook indices + 8 x 3-bit sub-scales packed in MSBs)
|
| 271 |
+
+typedef struct {
|
| 272 |
+
+ ggml_half d; // master FP16 super-block scale (2 bytes)
|
| 273 |
+
+ uint8_t qs[22]; // 21 x 7-bit 3D codebook indices + 1 outlier byte + 8x3-bit sub-scales (22 bytes)
|
| 274 |
+
+} block_vq3_0;
|
| 275 |
+
+static_assert(sizeof(block_vq3_0) == 24, "wrong vq3_0 block size/padding");
|
| 276 |
+
+
|
| 277 |
+
#define QK4_0 32
|
| 278 |
+
typedef struct {
|
| 279 |
+
ggml_half d; // delta
|
| 280 |
+
diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp
|
| 281 |
+
index 554a206..91d4856 100644
|
| 282 |
+
--- a/ggml/src/ggml-cpu/amx/amx.cpp
|
| 283 |
+
+++ b/ggml/src/ggml-cpu/amx/amx.cpp
|
| 284 |
+
@@ -152,6 +152,12 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
|
| 285 |
+
auto * src0 = op->src[0];
|
| 286 |
+
auto * src1 = op->src[1];
|
| 287 |
+
|
| 288 |
+
+ // only claim 2D weight tensors; 3D weights (e.g. MLA attn_k_b/attn_v_b) may be used
|
| 289 |
+
+ // with permuted src1 views that this backend cannot handle, which would leave the op
|
| 290 |
+
+ // with no supporting backend if the tensor was placed in the AMX buffer
|
| 291 |
+
+ if (ggml_n_dims(src0) != 2) {
|
| 292 |
+
+ return false;
|
| 293 |
+
+ }
|
| 294 |
+
if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {
|
| 295 |
+
return false;
|
| 296 |
+
}
|
| 297 |
+
diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h
|
| 298 |
+
index 4e59d0a..d2fb897 100644
|
| 299 |
+
--- a/ggml/src/ggml-cpu/arch-fallback.h
|
| 300 |
+
+++ b/ggml/src/ggml-cpu/arch-fallback.h
|
| 301 |
+
@@ -18,6 +18,7 @@
|
| 302 |
+
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
|
| 303 |
+
#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0
|
| 304 |
+
#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0
|
| 305 |
+
+#define ggml_vec_dot_vq3_0_q8_0_generic ggml_vec_dot_vq3_0_q8_0 // [llama-modchip port]
|
| 306 |
+
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
|
| 307 |
+
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
|
| 308 |
+
#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K
|
| 309 |
+
@@ -76,6 +77,8 @@
|
| 310 |
+
#define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0
|
| 311 |
+
#define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0
|
| 312 |
+
#elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC)
|
| 313 |
+
+// quants.c
|
| 314 |
+
+#define ggml_vec_dot_vq3_0_q8_0_generic ggml_vec_dot_vq3_0_q8_0 // [llama-modchip port] (generic scalar fallback provides the symbol on ARM)
|
| 315 |
+
// repack.cpp
|
| 316 |
+
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
|
| 317 |
+
#define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8
|
| 318 |
+
@@ -86,6 +89,8 @@
|
| 319 |
+
#define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0
|
| 320 |
+
#define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K
|
| 321 |
+
#elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)
|
| 322 |
+
+// quants.c
|
| 323 |
+
+#define ggml_vec_dot_vq3_0_q8_0_generic ggml_vec_dot_vq3_0_q8_0 // [llama-modchip port] (AVX-512 VBMI fast path lives in the generic fn)
|
| 324 |
+
// repack.cpp
|
| 325 |
+
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
|
| 326 |
+
#define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4
|
| 327 |
+
@@ -122,6 +127,7 @@
|
| 328 |
+
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
|
| 329 |
+
#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0
|
| 330 |
+
#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0
|
| 331 |
+
+#define ggml_vec_dot_vq3_0_q8_0_generic ggml_vec_dot_vq3_0_q8_0 // [llama-modchip port]
|
| 332 |
+
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
|
| 333 |
+
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
|
| 334 |
+
#define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K
|
| 335 |
+
@@ -176,6 +182,7 @@
|
| 336 |
+
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
|
| 337 |
+
#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0
|
| 338 |
+
#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0
|
| 339 |
+
+#define ggml_vec_dot_vq3_0_q8_0_generic ggml_vec_dot_vq3_0_q8_0 // [llama-modchip port]
|
| 340 |
+
// repack.cpp
|
| 341 |
+
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
|
| 342 |
+
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
|
| 343 |
+
@@ -221,6 +228,7 @@
|
| 344 |
+
// quants.c
|
| 345 |
+
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
|
| 346 |
+
#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0
|
| 347 |
+
+#define ggml_vec_dot_vq3_0_q8_0_generic ggml_vec_dot_vq3_0_q8_0 // [llama-modchip port]
|
| 348 |
+
// repack.cpp
|
| 349 |
+
#define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1
|
| 350 |
+
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
|
| 351 |
+
@@ -266,6 +274,7 @@
|
| 352 |
+
#define quantize_row_q8_K_generic quantize_row_q8_K
|
| 353 |
+
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
|
| 354 |
+
#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0
|
| 355 |
+
+#define ggml_vec_dot_vq3_0_q8_0_generic ggml_vec_dot_vq3_0_q8_0 // [llama-modchip port]
|
| 356 |
+
#define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K
|
| 357 |
+
#define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K
|
| 358 |
+
#define ggml_vec_dot_q2_K_q8_K_generic ggml_vec_dot_q2_K_q8_K
|
| 359 |
+
@@ -331,6 +340,7 @@
|
| 360 |
+
#define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0
|
| 361 |
+
#define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0
|
| 362 |
+
#define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0
|
| 363 |
+
+#define ggml_vec_dot_vq3_0_q8_0_generic ggml_vec_dot_vq3_0_q8_0 // [llama-modchip port]
|
| 364 |
+
// repack.cpp
|
| 365 |
+
#define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4
|
| 366 |
+
#define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8
|
| 367 |
+
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
|
| 368 |
+
index a76af84..0cac56f 100644
|
| 369 |
+
--- a/ggml/src/ggml-cpu/ggml-cpu.c
|
| 370 |
+
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
|
| 371 |
+
@@ -240,6 +240,12 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
|
| 372 |
+
.vec_dot_type = GGML_TYPE_Q8_0,
|
| 373 |
+
.nrows = 1,
|
| 374 |
+
},
|
| 375 |
+
+ [GGML_TYPE_VQ3_0] = { // [llama-modchip port]
|
| 376 |
+
+ .from_float = quantize_row_vq3_0,
|
| 377 |
+
+ .vec_dot = ggml_vec_dot_vq3_0_q8_0,
|
| 378 |
+
+ .vec_dot_type = GGML_TYPE_Q8_0,
|
| 379 |
+
+ .nrows = 1,
|
| 380 |
+
+ },
|
| 381 |
+
[GGML_TYPE_Q4_0] = {
|
| 382 |
+
.from_float = quantize_row_q4_0,
|
| 383 |
+
.vec_dot = ggml_vec_dot_q4_0_q8_0,
|
| 384 |
+
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
|
| 385 |
+
index 7c25427..1baf10e 100644
|
| 386 |
+
--- a/ggml/src/ggml-cpu/ops.cpp
|
| 387 |
+
+++ b/ggml/src/ggml-cpu/ops.cpp
|
| 388 |
+
@@ -675,6 +675,7 @@ void ggml_compute_forward_add(
|
| 389 |
+
} break;
|
| 390 |
+
case GGML_TYPE_Q1_0:
|
| 391 |
+
case GGML_TYPE_Q2_0:
|
| 392 |
+
+ case GGML_TYPE_VQ3_0: // [llama-modchip port]
|
| 393 |
+
case GGML_TYPE_Q4_0:
|
| 394 |
+
case GGML_TYPE_Q4_1:
|
| 395 |
+
case GGML_TYPE_Q5_0:
|
| 396 |
+
@@ -1132,6 +1133,7 @@ void ggml_compute_forward_add1(
|
| 397 |
+
} break;
|
| 398 |
+
case GGML_TYPE_Q1_0:
|
| 399 |
+
case GGML_TYPE_Q2_0:
|
| 400 |
+
+ case GGML_TYPE_VQ3_0: // [llama-modchip port]
|
| 401 |
+
case GGML_TYPE_Q4_0:
|
| 402 |
+
case GGML_TYPE_Q4_1:
|
| 403 |
+
case GGML_TYPE_Q5_0:
|
| 404 |
+
@@ -1264,6 +1266,7 @@ void ggml_compute_forward_acc(
|
| 405 |
+
case GGML_TYPE_BF16:
|
| 406 |
+
case GGML_TYPE_Q1_0:
|
| 407 |
+
case GGML_TYPE_Q2_0:
|
| 408 |
+
+ case GGML_TYPE_VQ3_0: // [llama-modchip port]
|
| 409 |
+
case GGML_TYPE_Q4_0:
|
| 410 |
+
case GGML_TYPE_Q4_1:
|
| 411 |
+
case GGML_TYPE_Q5_0:
|
| 412 |
+
@@ -4857,6 +4860,7 @@ void ggml_compute_forward_out_prod(
|
| 413 |
+
switch (src0->type) {
|
| 414 |
+
case GGML_TYPE_Q1_0:
|
| 415 |
+
case GGML_TYPE_Q2_0:
|
| 416 |
+
+ case GGML_TYPE_VQ3_0: // [llama-modchip port]
|
| 417 |
+
case GGML_TYPE_Q4_0:
|
| 418 |
+
case GGML_TYPE_Q4_1:
|
| 419 |
+
case GGML_TYPE_Q5_0:
|
| 420 |
+
@@ -5182,6 +5186,7 @@ void ggml_compute_forward_set(
|
| 421 |
+
case GGML_TYPE_BF16:
|
| 422 |
+
case GGML_TYPE_Q1_0:
|
| 423 |
+
case GGML_TYPE_Q2_0:
|
| 424 |
+
+ case GGML_TYPE_VQ3_0: // [llama-modchip port]
|
| 425 |
+
case GGML_TYPE_Q4_0:
|
| 426 |
+
case GGML_TYPE_Q4_1:
|
| 427 |
+
case GGML_TYPE_Q5_0:
|
| 428 |
+
@@ -5408,6 +5413,7 @@ void ggml_compute_forward_get_rows(
|
| 429 |
+
switch (src0->type) {
|
| 430 |
+
case GGML_TYPE_Q1_0:
|
| 431 |
+
case GGML_TYPE_Q2_0:
|
| 432 |
+
+ case GGML_TYPE_VQ3_0: // [llama-modchip port]
|
| 433 |
+
case GGML_TYPE_Q4_0:
|
| 434 |
+
case GGML_TYPE_Q4_1:
|
| 435 |
+
case GGML_TYPE_Q5_0:
|
| 436 |
+
@@ -6081,6 +6087,7 @@ void ggml_compute_forward_clamp(
|
| 437 |
+
case GGML_TYPE_BF16:
|
| 438 |
+
case GGML_TYPE_Q1_0:
|
| 439 |
+
case GGML_TYPE_Q2_0:
|
| 440 |
+
+ case GGML_TYPE_VQ3_0: // [llama-modchip port]
|
| 441 |
+
case GGML_TYPE_Q4_0:
|
| 442 |
+
case GGML_TYPE_Q4_1:
|
| 443 |
+
case GGML_TYPE_Q5_0:
|
| 444 |
+
diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c
|
| 445 |
+
index 44ea9fa..c0d44b5 100644
|
| 446 |
+
--- a/ggml/src/ggml-cpu/quants.c
|
| 447 |
+
+++ b/ggml/src/ggml-cpu/quants.c
|
| 448 |
+
@@ -8,6 +8,7 @@
|
| 449 |
+
#include "quants.h"
|
| 450 |
+
|
| 451 |
+
#include "arch-fallback.h"
|
| 452 |
+
+#include "../vq3_codebook.h" // LUT-VQ3-64 [llama-modchip port]
|
| 453 |
+
|
| 454 |
+
#include <string.h>
|
| 455 |
+
#include <assert.h>
|
| 456 |
+
@@ -31,6 +32,113 @@ void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in
|
| 457 |
+
quantize_row_q2_0_ref(x, y, k);
|
| 458 |
+
}
|
| 459 |
+
|
| 460 |
+
+// ==== LUT-VQ3-64 [llama-modchip port] ========================================
|
| 461 |
+
+
|
| 462 |
+
+void quantize_row_vq3_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
|
| 463 |
+
+ quantize_row_vq3_0_ref(x, y, k);
|
| 464 |
+
+}
|
| 465 |
+
+
|
| 466 |
+
+#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VBMI__)
|
| 467 |
+
+#define GGML_VQ3_0_INIT_CONSTS() \
|
| 468 |
+
+ const __m512i zero512 = _mm512_setzero_si512(); \
|
| 469 |
+
+ const __m512i lut0_lo = ggml_vq3_loadu_si512(ggml_vq3_lut0_lo); \
|
| 470 |
+
+ const __m512i lut0_hi = ggml_vq3_loadu_si512(ggml_vq3_lut0_hi); \
|
| 471 |
+
+ const __m512i lut1_lo = ggml_vq3_loadu_si512(ggml_vq3_lut1_lo); \
|
| 472 |
+
+ const __m512i lut1_hi = ggml_vq3_loadu_si512(ggml_vq3_lut1_hi); \
|
| 473 |
+
+ const __m512i lut2_lo = ggml_vq3_loadu_si512(ggml_vq3_lut2_lo); \
|
| 474 |
+
+ const __m512i lut2_hi = ggml_vq3_loadu_si512(ggml_vq3_lut2_hi); \
|
| 475 |
+
+ const __m512i perm_rep64 = ggml_vq3_loadu_si512(ggml_vq3_perm_rep64); \
|
| 476 |
+
+ const __m512i shuf_mult512 = ggml_vq3_loadu_si512(ggml_vq3_shuf_mult512);
|
| 477 |
+
+
|
| 478 |
+
+#define GGML_VQ3_0_UNPACK_STEP(i) \
|
| 479 |
+
+ const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d) * GGML_VQ3_G; \
|
| 480 |
+
+ const __m512i raw_22 = _mm512_maskz_loadu_epi8(0x00000000003FFFFFULL, x[i].qs); \
|
| 481 |
+
+ const uint32_t s24 = (uint32_t)_mm512_movepi8_mask(raw_22) | \
|
| 482 |
+
+ (((uint32_t)(x[i].qs[20] >> 6) & 1U) << 22) | \
|
| 483 |
+
+ (((uint32_t)(x[i].qs[21] >> 6) & 1U) << 23); \
|
| 484 |
+
+ const uint64_t m_all = ggml_vq3_unpack_scales(s24); \
|
| 485 |
+
+ const __m512i vmult = _mm512_shuffle_epi8(_mm512_set1_epi64((int64_t)m_all), shuf_mult512); \
|
| 486 |
+
+ __m512i t_idx = _mm512_permutexvar_epi8(perm_rep64, raw_22); \
|
| 487 |
+
+ t_idx = _mm512_mask2_permutex2var_epi8(lut1_lo, t_idx, 0x00000FFFFFC00000ULL, lut1_hi); \
|
| 488 |
+
+ t_idx = _mm512_mask2_permutex2var_epi8(lut2_lo, t_idx, 0xFFFFF00000000000ULL, lut2_hi); \
|
| 489 |
+
+ const __m512i q_s8 = _mm512_mask2_permutex2var_epi8(lut0_lo, t_idx, 0x00000000003FFFFFULL, lut0_hi); \
|
| 490 |
+
+ const __m512i q_abs = _mm512_abs_epi8(q_s8); \
|
| 491 |
+
+ const __mmask64 neg_mask = _mm512_movepi8_mask(q_s8);
|
| 492 |
+
+
|
| 493 |
+
+#define GGML_VQ3_0_DOT_COL(Y_PTR, ACC) \
|
| 494 |
+
+ do { \
|
| 495 |
+
+ const __m512 vd = _mm512_mask_broadcastss_ps( \
|
| 496 |
+
+ _mm512_set1_ps(d0 * GGML_CPU_FP16_TO_FP32((Y_PTR)[2 * i + 0].d)), 0xFF00, \
|
| 497 |
+
+ _mm_set_ss(d0 * GGML_CPU_FP16_TO_FP32((Y_PTR)[2 * i + 1].d))); \
|
| 498 |
+
+ const __m512i q8_all = _mm512_inserti64x4( \
|
| 499 |
+
+ _mm512_castsi256_si512(_mm256_loadu_si256((const __m256i *)(Y_PTR)[2 * i + 0].qs)), \
|
| 500 |
+
+ _mm256_loadu_si256((const __m256i *)(Y_PTR)[2 * i + 1].qs), 1); \
|
| 501 |
+
+ const __m512i q8_signed = _mm512_mask_sub_epi8(q8_all, neg_mask, zero512, q8_all); \
|
| 502 |
+
+ const __m512i p = _mm512_maddubs_epi16(q_abs, q8_signed); \
|
| 503 |
+
+ const __m512i s_i32 = _mm512_madd_epi16(p, vmult); \
|
| 504 |
+
+ (ACC) = _mm512_fmadd_ps(vd, _mm512_cvtepi32_ps(s_i32), (ACC)); \
|
| 505 |
+
+ } while (0)
|
| 506 |
+
+#endif
|
| 507 |
+
+
|
| 508 |
+
+void ggml_vec_dot_vq3_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
|
| 509 |
+
+ assert(n % QK_VQ3_0 == 0);
|
| 510 |
+
+ assert(nrc == 1);
|
| 511 |
+
+ UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs);
|
| 512 |
+
+ const int nb = n / QK_VQ3_0;
|
| 513 |
+
+ const block_vq3_0 * GGML_RESTRICT x = (const block_vq3_0 *)vx;
|
| 514 |
+
+ const block_q8_0 * GGML_RESTRICT y = (const block_q8_0 *)vy;
|
| 515 |
+
+#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VBMI__)
|
| 516 |
+
+ GGML_VQ3_0_INIT_CONSTS();
|
| 517 |
+
+ __m512 acc0 = _mm512_setzero_ps();
|
| 518 |
+
+ for (int i = 0; i < nb; ++i) {
|
| 519 |
+
+ GGML_VQ3_0_UNPACK_STEP(i);
|
| 520 |
+
+ GGML_VQ3_0_DOT_COL(y, acc0);
|
| 521 |
+
+ }
|
| 522 |
+
+ *s = _mm512_reduce_add_ps(acc0);
|
| 523 |
+
+#else
|
| 524 |
+
+ float sumf = 0.0f;
|
| 525 |
+
+ for (int i = 0; i < nb; ++i) {
|
| 526 |
+
+ const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d) * GGML_VQ3_G;
|
| 527 |
+
+ uint32_t s24 = 0;
|
| 528 |
+
+ for (int j = 0; j < 22; ++j) {
|
| 529 |
+
+ if (x[i].qs[j] & 0x80U) s24 |= (1U << j);
|
| 530 |
+
+ }
|
| 531 |
+
+ s24 |= (((uint32_t)(x[i].qs[20] >> 6) & 1U) << 22);
|
| 532 |
+
+ s24 |= (((uint32_t)(x[i].qs[21] >> 6) & 1U) << 23);
|
| 533 |
+
+ int m[8];
|
| 534 |
+
+ for (int b = 0; b < 8; ++b) m[b] = (int)(((s24 >> (3 * b)) & 0x7U) + 2U);
|
| 535 |
+
+ int8_t qw[64];
|
| 536 |
+
+ for (int j = 0; j < 20; ++j) {
|
| 537 |
+
+ const int c = x[i].qs[j] & 0x7F;
|
| 538 |
+
+ qw[j] = ggml_vq3_cb[c][0];
|
| 539 |
+
+ qw[j + 22] = ggml_vq3_cb[c][1];
|
| 540 |
+
+ qw[j + 44] = ggml_vq3_cb[c][2];
|
| 541 |
+
+ }
|
| 542 |
+
+ for (int j = 20; j < 22; ++j) {
|
| 543 |
+
+ const int c = x[i].qs[j] & 0x7F;
|
| 544 |
+
+ qw[j] = ggml_vq3_cb[c][0];
|
| 545 |
+
+ qw[j + 22] = ggml_vq3_cb[c][1];
|
| 546 |
+
+ }
|
| 547 |
+
+ int32_t sum0 = 0, sum1 = 0;
|
| 548 |
+
+ for (int b = 0; b < 4; ++b) {
|
| 549 |
+
+ int32_t sb = 0;
|
| 550 |
+
+ for (int j = 0; j < 8; ++j) sb += (int32_t)qw[b * 8 + j] * (int32_t)y[2 * i + 0].qs[b * 8 + j];
|
| 551 |
+
+ sum0 += sb * m[b];
|
| 552 |
+
+ }
|
| 553 |
+
+ for (int b = 4; b < 8; ++b) {
|
| 554 |
+
+ int32_t sb = 0;
|
| 555 |
+
+ for (int j = 0; j < 8; ++j) sb += (int32_t)qw[b * 8 + j] * (int32_t)y[2 * i + 1].qs[(b - 4) * 8 + j];
|
| 556 |
+
+ sum1 += sb * m[b];
|
| 557 |
+
+ }
|
| 558 |
+
+ sumf += d0 * (GGML_CPU_FP16_TO_FP32(y[2 * i + 0].d) * (float)sum0 +
|
| 559 |
+
+ GGML_CPU_FP16_TO_FP32(y[2 * i + 1].d) * (float)sum1);
|
| 560 |
+
+ }
|
| 561 |
+
+ *s = sumf;
|
| 562 |
+
+#endif
|
| 563 |
+
+}
|
| 564 |
+
+
|
| 565 |
+
+// ==== end LUT-VQ3-64 =========================================================
|
| 566 |
+
+
|
| 567 |
+
void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
|
| 568 |
+
quantize_row_q4_0_ref(x, y, k);
|
| 569 |
+
}
|
| 570 |
+
diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h
|
| 571 |
+
index f04be52..fb96cae 100644
|
| 572 |
+
--- a/ggml/src/ggml-cpu/quants.h
|
| 573 |
+
+++ b/ggml/src/ggml-cpu/quants.h
|
| 574 |
+
@@ -14,6 +14,7 @@ extern "C" {
|
| 575 |
+
// Quantization
|
| 576 |
+
void quantize_row_q1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
| 577 |
+
void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
| 578 |
+
+void quantize_row_vq3_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); // [llama-modchip port]
|
| 579 |
+
void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
| 580 |
+
void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
| 581 |
+
void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
| 582 |
+
@@ -41,6 +42,7 @@ void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y,
|
| 583 |
+
// Dot product
|
| 584 |
+
void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
| 585 |
+
void ggml_vec_dot_q2_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
| 586 |
+
+void ggml_vec_dot_vq3_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); // [llama-modchip port]
|
| 587 |
+
void ggml_vec_dot_f8_e4m3_f32(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
| 588 |
+
void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
| 589 |
+
void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
| 590 |
+
@@ -76,6 +78,7 @@ void quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRI
|
| 591 |
+
void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
| 592 |
+
void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
| 593 |
+
void ggml_vec_dot_q2_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
| 594 |
+
+void ggml_vec_dot_vq3_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); // [llama-modchip port]
|
| 595 |
+
void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
| 596 |
+
void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
| 597 |
+
void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
| 598 |
+
diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c
|
| 599 |
+
index 3586f67..db92c9a 100644
|
| 600 |
+
--- a/ggml/src/ggml-quants.c
|
| 601 |
+
+++ b/ggml/src/ggml-quants.c
|
| 602 |
+
@@ -2,6 +2,7 @@
|
| 603 |
+
#include "ggml-common.h"
|
| 604 |
+
|
| 605 |
+
#include "ggml-quants.h"
|
| 606 |
+
+#include "vq3_codebook.h" // LUT-VQ3-64 [llama-modchip port]
|
| 607 |
+
#include "ggml-impl.h"
|
| 608 |
+
#include "ggml-cpu/ggml-cpu-impl.h"
|
| 609 |
+
#include "ggml-cpu.h"
|
| 610 |
+
@@ -459,6 +460,164 @@ void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRI
|
| 611 |
+
}
|
| 612 |
+
}
|
| 613 |
+
|
| 614 |
+
+// ==== LUT-VQ3-64 [llama-modchip port] ========================================
|
| 615 |
+
+
|
| 616 |
+
+void quantize_row_vq3_0_ref(const float * GGML_RESTRICT x, block_vq3_0 * GGML_RESTRICT y, int64_t k) {
|
| 617 |
+
+ assert(k % QK_VQ3_0 == 0);
|
| 618 |
+
+ const int64_t nb = k / QK_VQ3_0;
|
| 619 |
+
+ float cb_q[128][3];
|
| 620 |
+
+ float cb_c2[128][3];
|
| 621 |
+
+ for (int c = 0; c < 128; ++c) {
|
| 622 |
+
+ for (int j = 0; j < 3; ++j) {
|
| 623 |
+
+ const float v = (float)ggml_vq3_cb[c][j] * GGML_VQ3_G;
|
| 624 |
+
+ cb_q[c][j] = v;
|
| 625 |
+
+ cb_c2[c][j] = v * v;
|
| 626 |
+
+ }
|
| 627 |
+
+ }
|
| 628 |
+
+
|
| 629 |
+
+ for (int64_t b = 0; b < nb; ++b) {
|
| 630 |
+
+ const float * xb = x + b * 64;
|
| 631 |
+
+ float sub_amax[8];
|
| 632 |
+
+ float blk_amax = 0.0f;
|
| 633 |
+
+ for (int s = 0; s < 8; ++s) {
|
| 634 |
+
+ float amax = 0.0f;
|
| 635 |
+
+ for (int j = 0; j < 8; ++j) {
|
| 636 |
+
+ const float av = fabsf(xb[s * 8 + j]);
|
| 637 |
+
+ if (av > amax) amax = av;
|
| 638 |
+
+ }
|
| 639 |
+
+ sub_amax[s] = amax * 0.88f;
|
| 640 |
+
+ if (sub_amax[s] > blk_amax) blk_amax = sub_amax[s];
|
| 641 |
+
+ }
|
| 642 |
+
+
|
| 643 |
+
+ const ggml_half d_h = (blk_amax > 0.0f) ? GGML_FP32_TO_FP16(blk_amax / 9.0f) : 0;
|
| 644 |
+
+ const float d = GGML_FP16_TO_FP32(d_h);
|
| 645 |
+
+ if (d <= 1e-30f) {
|
| 646 |
+
+ y[b].d = 0;
|
| 647 |
+
+ memset(y[b].qs, 0, 22);
|
| 648 |
+
+ continue;
|
| 649 |
+
+ }
|
| 650 |
+
+
|
| 651 |
+
+ float m[8];
|
| 652 |
+
+ for (int s = 0; s < 8; ++s) {
|
| 653 |
+
+ float mr = roundf(sub_amax[s] / d);
|
| 654 |
+
+ if (mr < 2.0f) mr = 2.0f;
|
| 655 |
+
+ if (mr > 9.0f) mr = 9.0f;
|
| 656 |
+
+ m[s] = mr;
|
| 657 |
+
+ }
|
| 658 |
+
+
|
| 659 |
+
+ uint8_t idx22[22];
|
| 660 |
+
+ float w_rec[64];
|
| 661 |
+
+
|
| 662 |
+
+ for (int iter = 0; iter < 3; ++iter) {
|
| 663 |
+
+ float eff_s[64];
|
| 664 |
+
+ for (int s = 0; s < 8; ++s) {
|
| 665 |
+
+ const float es = d * m[s];
|
| 666 |
+
+ for (int j = 0; j < 8; ++j) eff_s[s * 8 + j] = es;
|
| 667 |
+
+ }
|
| 668 |
+
+
|
| 669 |
+
+ for (int j = 0; j < 20; ++j) {
|
| 670 |
+
+ const float w0 = xb[j], s0 = eff_s[j], s02 = s0 * s0, ws0 = w0 * s0;
|
| 671 |
+
+ const float w1 = xb[j + 22], s1 = eff_s[j + 22], s12 = s1 * s1, ws1 = w1 * s1;
|
| 672 |
+
+ const float w2 = xb[j + 44], s2 = eff_s[j + 44], s22 = s2 * s2, ws2 = w2 * s2;
|
| 673 |
+
+ float best_d = 1e30f;
|
| 674 |
+
+ int best_c = 0;
|
| 675 |
+
+ for (int c = 0; c < 128; ++c) {
|
| 676 |
+
+ const float dist = (s02 * cb_c2[c][0] + s12 * cb_c2[c][1] + s22 * cb_c2[c][2])
|
| 677 |
+
+ - 2.0f * (ws0 * cb_q[c][0] + ws1 * cb_q[c][1] + ws2 * cb_q[c][2]);
|
| 678 |
+
+ if (dist < best_d) { best_d = dist; best_c = c; }
|
| 679 |
+
+ }
|
| 680 |
+
+ idx22[j] = (uint8_t)best_c;
|
| 681 |
+
+ w_rec[j] = cb_q[best_c][0];
|
| 682 |
+
+ w_rec[j + 22] = cb_q[best_c][1];
|
| 683 |
+
+ w_rec[j + 44] = cb_q[best_c][2];
|
| 684 |
+
+ }
|
| 685 |
+
+
|
| 686 |
+
+ const int s7_u3 = ((int)roundf(m[7] - 2.0f)) & 0x7;
|
| 687 |
+
+ for (int u = 0; u < 2; ++u) {
|
| 688 |
+
+ const int j = 20 + u;
|
| 689 |
+
+ const int bit_half = (s7_u3 >> (u + 1)) & 1;
|
| 690 |
+
+ const int c_start = bit_half ? 64 : 0;
|
| 691 |
+
+ const int c_end = bit_half ? 128 : 64;
|
| 692 |
+
+ const float w0 = xb[j], s0 = eff_s[j], s02 = s0 * s0, ws0 = w0 * s0;
|
| 693 |
+
+ const float w1 = xb[j + 22], s1 = eff_s[j + 22], s12 = s1 * s1, ws1 = w1 * s1;
|
| 694 |
+
+ float best_d = 1e30f;
|
| 695 |
+
+ int best_c = c_start;
|
| 696 |
+
+ for (int c = c_start; c < c_end; ++c) {
|
| 697 |
+
+ const float dist = (s02 * cb_c2[c][0] + s12 * cb_c2[c][1])
|
| 698 |
+
+ - 2.0f * (ws0 * cb_q[c][0] + ws1 * cb_q[c][1]);
|
| 699 |
+
+ if (dist < best_d) { best_d = dist; best_c = c; }
|
| 700 |
+
+ }
|
| 701 |
+
+ idx22[j] = (uint8_t)best_c;
|
| 702 |
+
+ w_rec[j] = cb_q[best_c][0];
|
| 703 |
+
+ w_rec[j + 22] = cb_q[best_c][1];
|
| 704 |
+
+ }
|
| 705 |
+
+
|
| 706 |
+
+ if (iter < 2) {
|
| 707 |
+
+ for (int s = 0; s < 8; ++s) {
|
| 708 |
+
+ float wq = 0.0f, qq = 0.0f;
|
| 709 |
+
+ for (int j = 0; j < 8; ++j) {
|
| 710 |
+
+ const float wv = xb[s * 8 + j];
|
| 711 |
+
+ const float qv = w_rec[s * 8 + j];
|
| 712 |
+
+ wq += wv * qv;
|
| 713 |
+
+ qq += qv * qv;
|
| 714 |
+
+ }
|
| 715 |
+
+ if (qq > 0.0f) {
|
| 716 |
+
+ float mo = roundf(wq / fmaxf(d * qq, 1e-30f));
|
| 717 |
+
+ if (mo < 2.0f) mo = 2.0f;
|
| 718 |
+
+ if (mo > 9.0f) mo = 9.0f;
|
| 719 |
+
+ m[s] = mo;
|
| 720 |
+
+ }
|
| 721 |
+
+ }
|
| 722 |
+
+ }
|
| 723 |
+
+ }
|
| 724 |
+
+
|
| 725 |
+
+ uint32_t s24 = 0;
|
| 726 |
+
+ for (int s = 0; s < 8; ++s) {
|
| 727 |
+
+ const uint32_t sv = ((uint32_t)roundf(m[s] - 2.0f)) & 0x7U;
|
| 728 |
+
+ s24 |= (sv << (3 * s));
|
| 729 |
+
+ }
|
| 730 |
+
+ y[b].d = d_h;
|
| 731 |
+
+ for (int k_i = 0; k_i < 22; ++k_i) {
|
| 732 |
+
+ const uint8_t msb = (uint8_t)((s24 >> k_i) & 1U);
|
| 733 |
+
+ y[b].qs[k_i] = (uint8_t)((idx22[k_i] & 0x7FU) | (msb << 7));
|
| 734 |
+
+ }
|
| 735 |
+
+ }
|
| 736 |
+
+}
|
| 737 |
+
+
|
| 738 |
+
+void dequantize_row_vq3_0(const block_vq3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
|
| 739 |
+
+ assert(k % QK_VQ3_0 == 0);
|
| 740 |
+
+ const int64_t nb = k / QK_VQ3_0;
|
| 741 |
+
+ for (int64_t b = 0; b < nb; ++b) {
|
| 742 |
+
+ const float d = GGML_FP16_TO_FP32(x[b].d) * GGML_VQ3_G;
|
| 743 |
+
+ uint32_t s24 = 0;
|
| 744 |
+
+ for (int j = 0; j < 22; ++j) {
|
| 745 |
+
+ if (x[b].qs[j] & 0x80U) s24 |= (1U << j);
|
| 746 |
+
+ }
|
| 747 |
+
+ s24 |= (((uint32_t)(x[b].qs[20] >> 6) & 1U) << 22);
|
| 748 |
+
+ s24 |= (((uint32_t)(x[b].qs[21] >> 6) & 1U) << 23);
|
| 749 |
+
+
|
| 750 |
+
+ float eff_s[64];
|
| 751 |
+
+ for (int s = 0; s < 8; ++s) {
|
| 752 |
+
+ const float ms = (float)(((s24 >> (3 * s)) & 0x7U) + 2U);
|
| 753 |
+
+ for (int j = 0; j < 8; ++j) eff_s[s * 8 + j] = d * ms;
|
| 754 |
+
+ }
|
| 755 |
+
+ float * yb = y + b * 64;
|
| 756 |
+
+ for (int j = 0; j < 20; ++j) {
|
| 757 |
+
+ const int c = x[b].qs[j] & 0x7F;
|
| 758 |
+
+ yb[j] = (float)ggml_vq3_cb[c][0] * eff_s[j];
|
| 759 |
+
+ yb[j + 22] = (float)ggml_vq3_cb[c][1] * eff_s[j + 22];
|
| 760 |
+
+ yb[j + 44] = (float)ggml_vq3_cb[c][2] * eff_s[j + 44];
|
| 761 |
+
+ }
|
| 762 |
+
+ for (int j = 20; j < 22; ++j) {
|
| 763 |
+
+ const int c = x[b].qs[j] & 0x7F;
|
| 764 |
+
+ yb[j] = (float)ggml_vq3_cb[c][0] * eff_s[j];
|
| 765 |
+
+ yb[j + 22] = (float)ggml_vq3_cb[c][1] * eff_s[j + 22];
|
| 766 |
+
+ }
|
| 767 |
+
+ }
|
| 768 |
+
+}
|
| 769 |
+
+
|
| 770 |
+
+// ==== end LUT-VQ3-64 ==========================================================
|
| 771 |
+
+
|
| 772 |
+
void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
|
| 773 |
+
static const int qk = QK4_0;
|
| 774 |
+
|
| 775 |
+
@@ -2135,6 +2294,12 @@ size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst,
|
| 776 |
+
return nrow * row_size;
|
| 777 |
+
}
|
| 778 |
+
|
| 779 |
+
+size_t quantize_vq3_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
|
| 780 |
+
+ GGML_UNUSED(quant_weights);
|
| 781 |
+
+ quantize_row_vq3_0_ref(src, dst, (int64_t)nrow * n_per_row);
|
| 782 |
+
+ return nrow * ggml_row_size(GGML_TYPE_VQ3_0, n_per_row);
|
| 783 |
+
+}
|
| 784 |
+
+
|
| 785 |
+
size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) {
|
| 786 |
+
if (!quant_weights) {
|
| 787 |
+
quantize_row_q4_0_ref(src, dst, (int64_t)nrow*n_per_row);
|
| 788 |
+
@@ -5525,6 +5690,10 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte
|
| 789 |
+
{
|
| 790 |
+
VALIDATE_ROW_DATA_D_F16_IMPL(block_q2_0, data, nb);
|
| 791 |
+
} break;
|
| 792 |
+
+ case GGML_TYPE_VQ3_0:
|
| 793 |
+
+ {
|
| 794 |
+
+ VALIDATE_ROW_DATA_D_F16_IMPL(block_vq3_0, data, nb);
|
| 795 |
+
+ } break;
|
| 796 |
+
case GGML_TYPE_Q4_0:
|
| 797 |
+
{
|
| 798 |
+
VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_0, data, nb);
|
| 799 |
+
diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h
|
| 800 |
+
index ca36d13..faffe4b 100644
|
| 801 |
+
--- a/ggml/src/ggml-quants.h
|
| 802 |
+
+++ b/ggml/src/ggml-quants.h
|
| 803 |
+
@@ -16,6 +16,7 @@ extern "C" {
|
| 804 |
+
// Quantization
|
| 805 |
+
GGML_API void quantize_row_q1_0_ref(const float * GGML_RESTRICT x, block_q1_0 * GGML_RESTRICT y, int64_t k);
|
| 806 |
+
GGML_API void quantize_row_q2_0_ref(const float * GGML_RESTRICT x, block_q2_0 * GGML_RESTRICT y, int64_t k);
|
| 807 |
+
+GGML_API void quantize_row_vq3_0_ref(const float * GGML_RESTRICT x, block_vq3_0 * GGML_RESTRICT y, int64_t k); // [llama-modchip port]
|
| 808 |
+
GGML_API void quantize_row_q4_0_ref(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
|
| 809 |
+
GGML_API void quantize_row_q4_1_ref(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
|
| 810 |
+
GGML_API void quantize_row_q5_0_ref(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
|
| 811 |
+
@@ -46,6 +47,7 @@ GGML_API void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_
|
| 812 |
+
// Dequantization
|
| 813 |
+
GGML_API void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
| 814 |
+
GGML_API void dequantize_row_q2_0(const block_q2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
| 815 |
+
+GGML_API void dequantize_row_vq3_0(const block_vq3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); // [llama-modchip port]
|
| 816 |
+
GGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
| 817 |
+
GGML_API void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
| 818 |
+
GGML_API void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
| 819 |
+
@@ -98,6 +100,7 @@ GGML_API size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTR
|
| 820 |
+
GGML_API size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
| 821 |
+
GGML_API size_t quantize_q1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
| 822 |
+
GGML_API size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
| 823 |
+
+GGML_API size_t quantize_vq3_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); // [llama-modchip port]
|
| 824 |
+
GGML_API size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
| 825 |
+
GGML_API size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
| 826 |
+
GGML_API size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
| 827 |
+
diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
|
| 828 |
+
index 1fca258..ab382d2 100644
|
| 829 |
+
--- a/ggml/src/ggml.c
|
| 830 |
+
+++ b/ggml/src/ggml.c
|
| 831 |
+
@@ -697,6 +697,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {
|
| 832 |
+
.to_float = (ggml_to_float_t) dequantize_row_q2_0,
|
| 833 |
+
.from_float_ref = (ggml_from_float_t) quantize_row_q2_0_ref,
|
| 834 |
+
},
|
| 835 |
+
+ [GGML_TYPE_VQ3_0] = { // [llama-modchip port]
|
| 836 |
+
+ .type_name = "vq3_0",
|
| 837 |
+
+ .blck_size = QK_VQ3_0,
|
| 838 |
+
+ .type_size = sizeof(block_vq3_0),
|
| 839 |
+
+ .is_quantized = true,
|
| 840 |
+
+ .to_float = (ggml_to_float_t) dequantize_row_vq3_0,
|
| 841 |
+
+ .from_float_ref = (ggml_from_float_t) quantize_row_vq3_0_ref,
|
| 842 |
+
+ },
|
| 843 |
+
[GGML_TYPE_Q4_0] = {
|
| 844 |
+
.type_name = "q4_0",
|
| 845 |
+
.blck_size = QK4_0,
|
| 846 |
+
@@ -1440,6 +1448,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) {
|
| 847 |
+
case GGML_FTYPE_MOSTLY_Q4_1: wtype = GGML_TYPE_Q4_1; break;
|
| 848 |
+
case GGML_FTYPE_MOSTLY_Q1_0: wtype = GGML_TYPE_Q1_0; break;
|
| 849 |
+
case GGML_FTYPE_MOSTLY_Q2_0: wtype = GGML_TYPE_Q2_0; break;
|
| 850 |
+
+ case GGML_FTYPE_MOSTLY_VQ3_0: wtype = GGML_TYPE_VQ3_0; break;
|
| 851 |
+
case GGML_FTYPE_MOSTLY_Q5_0: wtype = GGML_TYPE_Q5_0; break;
|
| 852 |
+
case GGML_FTYPE_MOSTLY_Q5_1: wtype = GGML_TYPE_Q5_1; break;
|
| 853 |
+
case GGML_FTYPE_MOSTLY_Q8_0: wtype = GGML_TYPE_Q8_0; break;
|
| 854 |
+
@@ -8028,6 +8037,7 @@ size_t ggml_quantize_chunk(
|
| 855 |
+
switch (type) {
|
| 856 |
+
case GGML_TYPE_Q1_0: result = quantize_q1_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
| 857 |
+
case GGML_TYPE_Q2_0: result = quantize_q2_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
| 858 |
+
+ case GGML_TYPE_VQ3_0: result = quantize_vq3_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
| 859 |
+
case GGML_TYPE_Q4_0: result = quantize_q4_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
| 860 |
+
case GGML_TYPE_Q4_1: result = quantize_q4_1 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
| 861 |
+
case GGML_TYPE_Q5_0: result = quantize_q5_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
| 862 |
+
diff --git a/include/llama.h b/include/llama.h
|
| 863 |
+
index 181d840..de86241 100644
|
| 864 |
+
--- a/include/llama.h
|
| 865 |
+
+++ b/include/llama.h
|
| 866 |
+
@@ -159,6 +159,7 @@ extern "C" {
|
| 867 |
+
LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors
|
| 868 |
+
LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors
|
| 869 |
+
LLAMA_FTYPE_MOSTLY_F8_E4M3 = 42, // except 1d tensors
|
| 870 |
+
+ LLAMA_FTYPE_MOSTLY_VQ3_0 = 43, // except 1d tensors [llama-modchip port]
|
| 871 |
+
|
| 872 |
+
LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
|
| 873 |
+
};
|
| 874 |
+
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
|
| 875 |
+
index 2cbee61..8bd9d22 100644
|
| 876 |
+
--- a/src/llama-model-loader.cpp
|
| 877 |
+
+++ b/src/llama-model-loader.cpp
|
| 878 |
+
@@ -41,6 +41,7 @@ const char * llama_ftype_name(llama_ftype ftype) {
|
| 879 |
+
case LLAMA_FTYPE_MOSTLY_Q1_0: name = LLAMA_FTYPE_PREFIX "Q1_0"; break;
|
| 880 |
+
case LLAMA_FTYPE_MOSTLY_Q2_0: name = LLAMA_FTYPE_PREFIX "Q2_0"; break;
|
| 881 |
+
case LLAMA_FTYPE_MOSTLY_F8_E4M3: name = LLAMA_FTYPE_PREFIX "F8_E4M3"; break;
|
| 882 |
+
+ case LLAMA_FTYPE_MOSTLY_VQ3_0: name = LLAMA_FTYPE_PREFIX "VQ3_0"; break; // [llama-modchip port]
|
| 883 |
+
case LLAMA_FTYPE_MOSTLY_Q4_0: name = LLAMA_FTYPE_PREFIX "Q4_0"; break;
|
| 884 |
+
case LLAMA_FTYPE_MOSTLY_Q4_1: name = LLAMA_FTYPE_PREFIX "Q4_1"; break;
|
| 885 |
+
case LLAMA_FTYPE_MOSTLY_Q5_0: name = LLAMA_FTYPE_PREFIX "Q5_0"; break;
|
| 886 |
+
@@ -867,6 +868,7 @@ llama_model_loader::llama_model_loader(
|
| 887 |
+
case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break;
|
| 888 |
+
case GGML_TYPE_Q2_0: ftype = LLAMA_FTYPE_MOSTLY_Q2_0; break;
|
| 889 |
+
case GGML_TYPE_F8_E4M3: ftype = LLAMA_FTYPE_MOSTLY_F8_E4M3; break;
|
| 890 |
+
+ case GGML_TYPE_VQ3_0: ftype = LLAMA_FTYPE_MOSTLY_VQ3_0; break; // [llama-modchip port]
|
| 891 |
+
default:
|
| 892 |
+
{
|
| 893 |
+
LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));
|
| 894 |
+
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
|
| 895 |
+
index 4ae3961..2d16400 100644
|
| 896 |
+
--- a/src/llama-quant.cpp
|
| 897 |
+
+++ b/src/llama-quant.cpp
|
| 898 |
+
@@ -470,6 +470,7 @@ static ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tenso
|
| 899 |
+
case GGML_TYPE_IQ3_S: // types on the right: block size 32
|
| 900 |
+
case GGML_TYPE_IQ4_XS: return_type = GGML_TYPE_IQ4_NL; break;
|
| 901 |
+
case GGML_TYPE_Q2_0:
|
| 902 |
+
+ case GGML_TYPE_VQ3_0: // [llama-modchip port]
|
| 903 |
+
case GGML_TYPE_Q2_K:
|
| 904 |
+
case GGML_TYPE_Q3_K:
|
| 905 |
+
case GGML_TYPE_TQ1_0:
|
| 906 |
+
@@ -605,7 +606,7 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type
|
| 907 |
+
else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {
|
| 908 |
+
new_type = GGML_TYPE_IQ3_S;
|
| 909 |
+
}
|
| 910 |
+
- else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0 || ftype == LLAMA_FTYPE_MOSTLY_Q2_0) {
|
| 911 |
+
+ else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0 || ftype == LLAMA_FTYPE_MOSTLY_Q2_0 || ftype == LLAMA_FTYPE_MOSTLY_VQ3_0) {
|
| 912 |
+
new_type = GGML_TYPE_Q4_K;
|
| 913 |
+
}
|
| 914 |
+
}
|
| 915 |
+
@@ -959,6 +960,7 @@ ggml_type llama_ftype_get_default_type(llama_ftype ftype) {
|
| 916 |
+
case LLAMA_FTYPE_ALL_F32: return GGML_TYPE_F32;
|
| 917 |
+
case LLAMA_FTYPE_MOSTLY_Q1_0: return GGML_TYPE_Q1_0;
|
| 918 |
+
case LLAMA_FTYPE_MOSTLY_Q2_0: return GGML_TYPE_Q2_0;
|
| 919 |
+
+ case LLAMA_FTYPE_MOSTLY_VQ3_0: return GGML_TYPE_VQ3_0; // [llama-modchip port]
|
| 920 |
+
|
| 921 |
+
case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return GGML_TYPE_MXFP4;
|
| 922 |
+
|
results-128k/rss128k_base_q8dq6.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-base-Q8dQ6.gguf -f ling-vq3/results-128k/prompt_120k.txt -c 131072 -n 64 -t 32 -tb 64 -ngl 0 --no-warmup --temp 0 -st"
|
| 5 |
+
User time (seconds): 168513.05
|
| 6 |
+
System time (seconds): 427.75
|
| 7 |
+
Percent of CPU this job got: 5840%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 48:12.64
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 9503968
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 0
|
| 16 |
+
Minor (reclaiming a frame) page faults: 26480932
|
| 17 |
+
Voluntary context switches: 1618960
|
| 18 |
+
Involuntary context switches: 648008
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 8565216
|
| 21 |
+
File system outputs: 24
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
results-128k/rss128k_bf16.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-bf16.gguf -f ling-vq3/results-128k/prompt_120k.txt -c 131072 -n 64 -t 32 -tb 64 -ngl 0 --no-warmup --temp 0 -st"
|
| 5 |
+
User time (seconds): 175113.79
|
| 6 |
+
System time (seconds): 232.56
|
| 7 |
+
Percent of CPU this job got: 5778%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 50:34.26
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 16723132
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 47
|
| 16 |
+
Minor (reclaiming a frame) page faults: 13696555
|
| 17 |
+
Voluntary context switches: 532788
|
| 18 |
+
Involuntary context switches: 3477546
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 20127880
|
| 21 |
+
File system outputs: 24
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
results-128k/rss128k_q4_k_m.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-Q4_K_M.gguf -f ling-vq3/results-128k/prompt_120k.txt -c 131072 -n 64 -t 32 -tb 64 -ngl 0 --no-warmup --temp 0 -st"
|
| 5 |
+
User time (seconds): 169304.84
|
| 6 |
+
System time (seconds): 497.68
|
| 7 |
+
Percent of CPU this job got: 5839%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 48:28.03
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 9418480
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 0
|
| 16 |
+
Minor (reclaiming a frame) page faults: 32885719
|
| 17 |
+
Voluntary context switches: 1000019
|
| 18 |
+
Involuntary context switches: 744727
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 4490976
|
| 21 |
+
File system outputs: 24
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
results-128k/rss128k_q8_0.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-Q8_0.gguf -f ling-vq3/results-128k/prompt_120k.txt -c 131072 -n 64 -t 32 -tb 64 -ngl 0 --no-warmup --temp 0 -st"
|
| 5 |
+
User time (seconds): 166981.97
|
| 6 |
+
System time (seconds): 494.81
|
| 7 |
+
Percent of CPU this job got: 5843%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 47:45.89
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 10052200
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 27
|
| 16 |
+
Minor (reclaiming a frame) page faults: 29573703
|
| 17 |
+
Voluntary context switches: 2217800
|
| 18 |
+
Involuntary context switches: 770195
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 10872
|
| 21 |
+
File system outputs: 24
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
results-128k/rss128k_vq3.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-VQ3.gguf -f ling-vq3/results-128k/prompt_120k.txt -c 131072 -n 64 -t 32 -tb 64 -ngl 0 --no-warmup --temp 0 -st"
|
| 5 |
+
User time (seconds): 172976.34
|
| 6 |
+
System time (seconds): 431.14
|
| 7 |
+
Percent of CPU this job got: 5871%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 49:13.55
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 6396264
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 19
|
| 16 |
+
Minor (reclaiming a frame) page faults: 28068370
|
| 17 |
+
Voluntary context switches: 913771
|
| 18 |
+
Involuntary context switches: 813915
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 48
|
| 21 |
+
File system outputs: 24
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
results-128k/rss128k_vq3_ub1024.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-VQ3.gguf -f ling-vq3/results-128k/prompt_120k.txt -c 131072 -n 64 -t 32 -tb 64 -ub 1024 -ngl 0 --no-warmup --temp 0 -st"
|
| 5 |
+
User time (seconds): 97375.78
|
| 6 |
+
System time (seconds): 487.80
|
| 7 |
+
Percent of CPU this job got: 5770%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 28:15.83
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 6576452
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 5
|
| 16 |
+
Minor (reclaiming a frame) page faults: 26888851
|
| 17 |
+
Voluntary context switches: 1787787
|
| 18 |
+
Involuntary context switches: 376187
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 1831888
|
| 21 |
+
File system outputs: 24
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
results-128k/rss128k_vq3_ub2048.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-VQ3.gguf -f ling-vq3/results-128k/prompt_120k.txt -c 131072 -n 64 -t 32 -tb 64 -ub 2048 -ngl 0 --no-warmup --temp 0 -st"
|
| 5 |
+
User time (seconds): 93958.58
|
| 6 |
+
System time (seconds): 2225.42
|
| 7 |
+
Percent of CPU this job got: 5724%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 28:00.17
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 6943248
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 0
|
| 16 |
+
Minor (reclaiming a frame) page faults: 75368197
|
| 17 |
+
Voluntary context switches: 2629393
|
| 18 |
+
Involuntary context switches: 352879
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 0
|
| 21 |
+
File system outputs: 24
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
results-128k/run_base_q8dq6.txt
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
|
| 3 |
+
Loading model... |-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/
|
| 4 |
+
|
| 5 |
+
▄▄ ▄▄
|
| 6 |
+
██ ██
|
| 7 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 8 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 9 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 10 |
+
██ ██
|
| 11 |
+
▀▀ ▀▀
|
| 12 |
+
|
| 13 |
+
build : b1-61dbf49
|
| 14 |
+
model : models/Ling-3.0-tiny-base-Q8dQ6.gguf
|
| 15 |
+
ftype : Q8_0
|
| 16 |
+
modalities : text
|
| 17 |
+
|
| 18 |
+
available commands:
|
| 19 |
+
/exit or Ctrl+C stop or exit
|
| 20 |
+
/regen regenerate the last response
|
| 21 |
+
/clear clear the chat history
|
| 22 |
+
/read <file> add a text file
|
| 23 |
+
/glob <pattern> add text files using globbing pattern
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
>
|
| 28 |
+
= Valkyria Chronicles III =
|
| 29 |
+
|
| 30 |
+
Senjō no Valkyria 3 : Unrecorded Chronicles ( Japanese : 戦場のヴァルキュリア3 , lit . Valkyria of the Battlefield 3 ) , commonly referred to as Valkyria Chronicles III outside Japan , is a tactical role @-@ playing video game developed by Sega and Media.Vision for the PlayStation Portable . Released in January 2011 in Japan , it is the third game in the Valkyria series . Employing the same fusion of tactical and real @-@ time gameplay as its predece ... (truncated)
|
| 31 |
+
|
| 32 |
+
[Start thinking]
|
| 33 |
+
|
| 34 |
+
The question asks me to find the answer from the provided text. The question is: "Who was the first person to be mentioned in the text?"
|
| 35 |
+
|
| 36 |
+
Let me scan through the text to find the very first person mentioned.
|
| 37 |
+
|
| 38 |
+
The text begins with "Senjō no Valkyria 3"
|
| 39 |
+
|
| 40 |
+
[ Prompt: 41.6 t/s | Generation: 6.0 t/s ]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
Exiting...
|
results-128k/run_bf16.txt
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
|
| 3 |
+
Loading model... |-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\
|
| 4 |
+
|
| 5 |
+
▄▄ ▄▄
|
| 6 |
+
██ ██
|
| 7 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 8 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 9 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 10 |
+
██ ██
|
| 11 |
+
▀▀ ▀▀
|
| 12 |
+
|
| 13 |
+
build : b1-61dbf49
|
| 14 |
+
model : models/Ling-3.0-tiny-bf16.gguf
|
| 15 |
+
ftype : BF16
|
| 16 |
+
modalities : text
|
| 17 |
+
|
| 18 |
+
available commands:
|
| 19 |
+
/exit or Ctrl+C stop or exit
|
| 20 |
+
/regen regenerate the last response
|
| 21 |
+
/clear clear the chat history
|
| 22 |
+
/read <file> add a text file
|
| 23 |
+
/glob <pattern> add text files using globbing pattern
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
>
|
| 28 |
+
= Valkyria Chronicles III =
|
| 29 |
+
|
| 30 |
+
Senjō no Valkyria 3 : Unrecorded Chronicles ( Japanese : 戦場のヴァルキュリア3 , lit . Valkyria of the Battlefield 3 ) , commonly referred to as Valkyria Chronicles III outside Japan , is a tactical role @-@ playing video game developed by Sega and Media.Vision for the PlayStation Portable . Released in January 2011 in Japan , it is the third game in the Valkyria series . Employing the same fusion of tactical and real @-@ time gameplay as its predece ... (truncated)
|
| 31 |
+
|
| 32 |
+
[Start thinking]
|
| 33 |
+
|
| 34 |
+
The question asks me to synthesize information from the provided text about the game "Senjō no Valkyria Chronicles 3" (Valkyria Chronicles III). Let me identify the key facts needed:
|
| 35 |
+
|
| 36 |
+
1. **Developer and Platform**: Developed by Sega and Media.Vision for PlayStation Portable.
|
| 37 |
+
|
| 38 |
+
[ Prompt: 39.7 t/s | Generation: 5.8 t/s ]
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
Exiting...
|
results-128k/run_q4_k_m.txt
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
|
| 3 |
+
Loading model... |-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\
|
| 4 |
+
|
| 5 |
+
▄▄ ▄▄
|
| 6 |
+
██ ██
|
| 7 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 8 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 9 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 10 |
+
██ ██
|
| 11 |
+
▀▀ ▀▀
|
| 12 |
+
|
| 13 |
+
build : b1-61dbf49
|
| 14 |
+
model : models/Ling-3.0-tiny-Q4_K_M.gguf
|
| 15 |
+
ftype : Q4_K - Medium
|
| 16 |
+
modalities : text
|
| 17 |
+
|
| 18 |
+
available commands:
|
| 19 |
+
/exit or Ctrl+C stop or exit
|
| 20 |
+
/regen regenerate the last response
|
| 21 |
+
/clear clear the chat history
|
| 22 |
+
/read <file> add a text file
|
| 23 |
+
/glob <pattern> add text files using globbing pattern
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
>
|
| 28 |
+
= Valkyria Chronicles III =
|
| 29 |
+
|
| 30 |
+
Senjō no Valkyria 3 : Unrecorded Chronicles ( Japanese : 戦場のヴァルキュリア3 , lit . Valkyria of the Battlefield 3 ) , commonly referred to as Valkyria Chronicles III outside Japan , is a tactical role @-@ playing video game developed by Sega and Media.Vision for the PlayStation Portable . Released in January 2011 in Japan , it is the third game in the Valkyria series . Employing the same fusion of tactical and real @-@ time gameplay as its predece ... (truncated)
|
| 31 |
+
|
| 32 |
+
[Start thinking]
|
| 33 |
+
|
| 34 |
+
The question asks about the "2011 – 12 Columbus Blue Jackets season" and specifically what the team's record was.
|
| 35 |
+
|
| 36 |
+
Let me find the relevant passage in the text:
|
| 37 |
+
|
| 38 |
+
"The 2011 – 12 Columbus Blue Jackets season was the team's 12th
|
| 39 |
+
|
| 40 |
+
[ Prompt: 41.4 t/s | Generation: 6.5 t/s ]
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
Exiting...
|
results-128k/run_q8_0.txt
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
|
| 3 |
+
Loading model... |-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\
|
| 4 |
+
|
| 5 |
+
▄▄ ▄▄
|
| 6 |
+
██ ██
|
| 7 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 8 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 9 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 10 |
+
██ ██
|
| 11 |
+
▀▀ ▀▀
|
| 12 |
+
|
| 13 |
+
build : b1-61dbf49
|
| 14 |
+
model : models/Ling-3.0-tiny-Q8_0.gguf
|
| 15 |
+
ftype : Q8_0
|
| 16 |
+
modalities : text
|
| 17 |
+
|
| 18 |
+
available commands:
|
| 19 |
+
/exit or Ctrl+C stop or exit
|
| 20 |
+
/regen regenerate the last response
|
| 21 |
+
/clear clear the chat history
|
| 22 |
+
/read <file> add a text file
|
| 23 |
+
/glob <pattern> add text files using globbing pattern
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
>
|
| 28 |
+
= Valkyria Chronicles III =
|
| 29 |
+
|
| 30 |
+
Senjō no Valkyria 3 : Unrecorded Chronicles ( Japanese : 戦場のヴァルキュリア3 , lit . Valkyria of the Battlefield 3 ) , commonly referred to as Valkyria Chronicles III outside Japan , is a tactical role @-@ playing video game developed by Sega and Media.Vision for the PlayStation Portable . Released in January 2011 in Japan , it is the third game in the Valkyria series . Employing the same fusion of tactical and real @-@ time gameplay as its predece ... (truncated)
|
| 31 |
+
|
| 32 |
+
[Start thinking]
|
| 33 |
+
|
| 34 |
+
The question asks about the "2011-12 Columbus Blue Jackets season" and specifically about the team's record, performance, and key events during that season.
|
| 35 |
+
|
| 36 |
+
Let me systematically extract all relevant information from the provided text about the 2011-12 Columbus Blue Jackets season.
|
| 37 |
+
|
| 38 |
+
[ Prompt: 41.9 t/s | Generation: 6.2 t/s ]
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
Exiting...
|
results-128k/run_vq3.txt
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
|
| 3 |
+
Loading model... |-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|
|
| 4 |
+
|
| 5 |
+
▄▄ ▄▄
|
| 6 |
+
██ ██
|
| 7 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 8 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 9 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 10 |
+
██ ██
|
| 11 |
+
▀▀ ▀▀
|
| 12 |
+
|
| 13 |
+
build : b1-61dbf49
|
| 14 |
+
model : models/Ling-3.0-tiny-VQ3.gguf
|
| 15 |
+
ftype : Q8_0
|
| 16 |
+
modalities : text
|
| 17 |
+
|
| 18 |
+
available commands:
|
| 19 |
+
/exit or Ctrl+C stop or exit
|
| 20 |
+
/regen regenerate the last response
|
| 21 |
+
/clear clear the chat history
|
| 22 |
+
/read <file> add a text file
|
| 23 |
+
/glob <pattern> add text files using globbing pattern
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
>
|
| 28 |
+
= Valkyria Chronicles III =
|
| 29 |
+
|
| 30 |
+
Senjō no Valkyria 3 : Unrecorded Chronicles ( Japanese : 戦場のヴァルキュリア3 , lit . Valkyria of the Battlefield 3 ) , commonly referred to as Valkyria Chronicles III outside Japan , is a tactical role @-@ playing video game developed by Sega and Media.Vision for the PlayStation Portable . Released in January 2011 in Japan , it is the third game in the Valkyria series . Employing the same fusion of tactical and real @-@ time gameplay as its predece ... (truncated)
|
| 31 |
+
|
| 32 |
+
[Start thinking]
|
| 33 |
+
|
| 34 |
+
The user has provided a massive block of text that contains multiple articles and sections on various topics. The text is structured with many "= =" section headers and covers topics like Valkyria Chronicles III, game gameplay, plot, development, music, release, reception, legacy, adaptations, the Tower Building of the
|
| 35 |
+
|
| 36 |
+
[ Prompt: 40.7 t/s | Generation: 6.5 t/s ]
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
Exiting...
|
results-128k/run_vq3_ub1024.txt
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
|
| 3 |
+
Loading model... |-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\|/
|
| 4 |
+
|
| 5 |
+
▄▄ ▄▄
|
| 6 |
+
██ ██
|
| 7 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 8 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 9 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 10 |
+
██ ██
|
| 11 |
+
▀▀ ▀▀
|
| 12 |
+
|
| 13 |
+
build : b1-61dbf49
|
| 14 |
+
model : models/Ling-3.0-tiny-VQ3.gguf
|
| 15 |
+
ftype : Q8_0
|
| 16 |
+
modalities : text
|
| 17 |
+
|
| 18 |
+
available commands:
|
| 19 |
+
/exit or Ctrl+C stop or exit
|
| 20 |
+
/regen regenerate the last response
|
| 21 |
+
/clear clear the chat history
|
| 22 |
+
/read <file> add a text file
|
| 23 |
+
/glob <pattern> add text files using globbing pattern
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
>
|
| 28 |
+
= Valkyria Chronicles III =
|
| 29 |
+
|
| 30 |
+
Senjō no Valkyria 3 : Unrecorded Chronicles ( Japanese : 戦場のヴァルキュリア3 , lit . Valkyria of the Battlefield 3 ) , commonly referred to as Valkyria Chronicles III outside Japan , is a tactical role @-@ playing video game developed by Sega and Media.Vision for the PlayStation Portable . Released in January 2011 in Japan , it is the third game in the Valkyria series . Employing the same fusion of tactical and real @-@ time gameplay as its predece ... (truncated)
|
| 31 |
+
|
| 32 |
+
[Start thinking]
|
| 33 |
+
|
| 34 |
+
The user has provided a massive block of text that contains multiple articles and sections on various topics. The text is structured with many "=" delimiters separating different sections. I need to identify what the user is asking for.
|
| 35 |
+
|
| 36 |
+
Looking at the structure, the text contains:
|
| 37 |
+
1. Valkyria Chronicles
|
| 38 |
+
|
| 39 |
+
[ Prompt: 71.2 t/s | Generation: 6.0 t/s ]
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
Exiting...
|
results-128k/run_vq3_ub2048.txt
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
|
| 3 |
+
Loading model... |-\|/-\|/-\|/-\|/-\|/-\|/-\|/-\
|
| 4 |
+
|
| 5 |
+
▄▄ ▄▄
|
| 6 |
+
██ ██
|
| 7 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 8 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 9 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 10 |
+
██ ██
|
| 11 |
+
▀▀ ▀▀
|
| 12 |
+
|
| 13 |
+
build : b1-61dbf49
|
| 14 |
+
model : models/Ling-3.0-tiny-VQ3.gguf
|
| 15 |
+
ftype : Q8_0
|
| 16 |
+
modalities : text
|
| 17 |
+
|
| 18 |
+
available commands:
|
| 19 |
+
/exit or Ctrl+C stop or exit
|
| 20 |
+
/regen regenerate the last response
|
| 21 |
+
/clear clear the chat history
|
| 22 |
+
/read <file> add a text file
|
| 23 |
+
/glob <pattern> add text files using globbing pattern
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
>
|
| 28 |
+
= Valkyria Chronicles III =
|
| 29 |
+
|
| 30 |
+
Senjō no Valkyria 3 : Unrecorded Chronicles ( Japanese : 戦場のヴァルキュリア3 , lit . Valkyria of the Battlefield 3 ) , commonly referred to as Valkyria Chronicles III outside Japan , is a tactical role @-@ playing video game developed by Sega and Media.Vision for the PlayStation Portable . Released in January 2011 in Japan , it is the third game in the Valkyria series . Employing the same fusion of tactical and real @-@ time gameplay as its predece ... (truncated)
|
| 31 |
+
|
| 32 |
+
[Start thinking]
|
| 33 |
+
|
| 34 |
+
The user has provided a massive block of text that contains multiple articles and sections on various topics. The text is structured with many "=" delimiters separating different sections. Let me analyze what's being asked.
|
| 35 |
+
|
| 36 |
+
Looking at the structure, the text covers:
|
| 37 |
+
1. Valkyria Chronicles III (a
|
| 38 |
+
|
| 39 |
+
[ Prompt: 71.8 t/s | Generation: 6.1 t/s ]
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
Exiting...
|
results-128k/tuning.tsv
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
=== numa off ===
|
| 2 |
+
| model | size | params | backend | threads | test | t/s |
|
| 3 |
+
| ------------------------------ | ---------: | ---------: | ---------- | ------: | --------------: | -------------------: |
|
| 4 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 32 | pp512 | 268.87 ± 0.00 |
|
| 5 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 32 | tg128 | 37.99 ± 0.00 |
|
| 6 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 48 | pp512 | 344.49 ± 0.00 |
|
| 7 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 48 | tg128 | 35.20 ± 0.00 |
|
| 8 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 64 | pp512 | 353.29 ± 0.00 |
|
| 9 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 64 | tg128 | 33.81 ± 0.00 |
|
| 10 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 80 | pp512 | 68.26 ± 0.00 |
|
| 11 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 80 | tg128 | 0.27 ± 0.00 |
|
| 12 |
+
|
| 13 |
+
build: 61dbf49 (1)
|
| 14 |
+
=== numa distribute ===
|
| 15 |
+
| model | size | params | backend | threads | test | t/s |
|
| 16 |
+
| ------------------------------ | ---------: | ---------: | ---------- | ------: | --------------: | -------------------: |
|
| 17 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 32 | pp512 | 266.22 ± 0.00 |
|
| 18 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 32 | tg128 | 38.89 ± 0.00 |
|
| 19 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 48 | pp512 | 270.80 ± 0.00 |
|
| 20 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 48 | tg128 | 33.51 ± 0.00 |
|
| 21 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 64 | pp512 | 335.81 ± 0.00 |
|
| 22 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 64 | tg128 | 32.41 ± 0.00 |
|
| 23 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 80 | pp512 | 76.73 ± 0.00 |
|
| 24 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 80 | tg128 | 0.87 ± 0.00 |
|
| 25 |
+
|
| 26 |
+
build: 61dbf49 (1)
|
results/bench.tsv
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
| model | size | params | backend | threads | test | t/s |
|
| 2 |
+
| ------------------------------ | ---------: | ---------: | ---------- | ------: | --------------: | -------------------: |
|
| 3 |
+
| bailingmoe3 7.9B.A1.3B BF16 | 14.71 GiB | 7.89 B | CPU | 64 | pp512 | 425.22 ± 19.90 |
|
| 4 |
+
| bailingmoe3 7.9B.A1.3B BF16 | 14.71 GiB | 7.89 B | CPU | 64 | tg128 | 26.89 ± 0.06 |
|
| 5 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 7.82 GiB | 7.89 B | CPU | 64 | pp512 | 503.99 ± 14.85 |
|
| 6 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 7.82 GiB | 7.89 B | CPU | 64 | tg128 | 32.99 ± 1.51 |
|
| 7 |
+
| bailingmoe3 7.9B.A1.3B Q4_K - Medium | 4.49 GiB | 7.89 B | CPU | 64 | pp512 | 424.16 ± 1.63 |
|
| 8 |
+
| bailingmoe3 7.9B.A1.3B Q4_K - Medium | 4.49 GiB | 7.89 B | CPU | 64 | tg128 | 33.70 ± 0.84 |
|
| 9 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 7.30 GiB | 7.89 B | CPU | 64 | pp512 | 457.76 ± 3.86 |
|
| 10 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 7.30 GiB | 7.89 B | CPU | 64 | tg128 | 34.19 ± 0.35 |
|
| 11 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 64 | pp512 | 368.65 ± 2.86 |
|
| 12 |
+
| bailingmoe3 7.9B.A1.3B Q8_0 | 4.34 GiB | 7.89 B | CPU | 64 | tg128 | 32.84 ± 1.06 |
|
| 13 |
+
|
| 14 |
+
build: 61dbf49 (1)
|
results/build_vq3.log
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[*] Compiling /mnt/disk3/work/21434/temp/work1/A/llama-modchip/vq3/vq3_enc_lib.c -> /mnt/disk3/work/21434/temp/work1/A/llama-modchip/vq3/libvq3_enc.so ...
|
| 2 |
+
Parsed headers: 526 tensors, encoding 46 expert tensors to VQ3_0(44), output size ~4.34 GiB
|
| 3 |
+
[ 1/46] blk.1.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.48s
|
| 4 |
+
[ 2/46] blk.1.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.18s
|
| 5 |
+
[ 3/46] blk.2.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.15s
|
| 6 |
+
[ 4/46] blk.2.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.15s
|
| 7 |
+
[ 5/46] blk.3.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.18s
|
| 8 |
+
[ 6/46] blk.3.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.18s
|
| 9 |
+
[ 7/46] blk.4.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.15s
|
| 10 |
+
[ 8/46] blk.4.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.16s
|
| 11 |
+
[ 9/46] blk.5.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.16s
|
| 12 |
+
[10/46] blk.5.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 13 |
+
[11/46] blk.6.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.15s
|
| 14 |
+
[12/46] blk.6.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.15s
|
| 15 |
+
[13/46] blk.7.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.15s
|
| 16 |
+
[14/46] blk.7.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.17s
|
| 17 |
+
[15/46] blk.8.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.16s
|
| 18 |
+
[16/46] blk.8.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 19 |
+
[17/46] blk.9.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 20 |
+
[18/46] blk.9.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 21 |
+
[19/46] blk.10.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 22 |
+
[20/46] blk.10.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 23 |
+
[21/46] blk.11.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.15s
|
| 24 |
+
[22/46] blk.11.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 25 |
+
[23/46] blk.12.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 26 |
+
[24/46] blk.12.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.16s
|
| 27 |
+
[25/46] blk.13.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.16s
|
| 28 |
+
[26/46] blk.13.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 29 |
+
[27/46] blk.14.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 30 |
+
[28/46] blk.14.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 31 |
+
[29/46] blk.15.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 32 |
+
[30/46] blk.15.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 33 |
+
[31/46] blk.16.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.16s
|
| 34 |
+
[32/46] blk.16.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 35 |
+
[33/46] blk.17.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 36 |
+
[34/46] blk.17.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 37 |
+
[35/46] blk.18.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 38 |
+
[36/46] blk.18.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 39 |
+
[37/46] blk.19.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 40 |
+
[38/46] blk.19.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 41 |
+
[39/46] blk.20.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.16s
|
| 42 |
+
[40/46] blk.20.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 43 |
+
[41/46] blk.21.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 44 |
+
[42/46] blk.21.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 45 |
+
[43/46] blk.22.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 46 |
+
[44/46] blk.22.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 47 |
+
[45/46] blk.23.ffn_gate_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.14s
|
| 48 |
+
[46/46] blk.23.ffn_up_exps.weight (100.7M w) BF16 -> VQ3_0 in 0.13s
|
| 49 |
+
DONE in 11.4s: models/Ling-3.0-tiny-VQ3.gguf (4.34 GiB)
|
results/convert_std.log
ADDED
|
@@ -0,0 +1,733 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
INFO:hf-to-gguf:Loading model: model
|
| 2 |
+
WARNING:hf-to-gguf:Failed to load model config from hf-ling/model: The repository hf-ling/model contains custom code which must be executed to correctly load the model. You can inspect the repository content at /mnt/disk3/work/21434/temp/work1/A/hf-ling/model .
|
| 3 |
+
You can inspect the repository content at https://hf.co/hf-ling/model.
|
| 4 |
+
Please pass the argument `trust_remote_code=True` to allow custom code to be run.
|
| 5 |
+
WARNING:hf-to-gguf:Trying to load config.json instead
|
| 6 |
+
INFO:hf-to-gguf:Model architecture: BailingMoeV3ForCausalLM
|
| 7 |
+
WARNING:hf-to-gguf:Failed to load model config from hf-ling/model: The repository hf-ling/model contains custom code which must be executed to correctly load the model. You can inspect the repository content at /mnt/disk3/work/21434/temp/work1/A/hf-ling/model .
|
| 8 |
+
You can inspect the repository content at https://hf.co/hf-ling/model.
|
| 9 |
+
Please pass the argument `trust_remote_code=True` to allow custom code to be run.
|
| 10 |
+
WARNING:hf-to-gguf:Trying to load config.json instead
|
| 11 |
+
INFO:hf-to-gguf:gguf: loading model weight map from 'model.safetensors.index.json'
|
| 12 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00000-of-00032.safetensors'
|
| 13 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00001-of-00032.safetensors'
|
| 14 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00002-of-00032.safetensors'
|
| 15 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00003-of-00032.safetensors'
|
| 16 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00004-of-00032.safetensors'
|
| 17 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00005-of-00032.safetensors'
|
| 18 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00006-of-00032.safetensors'
|
| 19 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00007-of-00032.safetensors'
|
| 20 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00008-of-00032.safetensors'
|
| 21 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00009-of-00032.safetensors'
|
| 22 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00010-of-00032.safetensors'
|
| 23 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00011-of-00032.safetensors'
|
| 24 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00012-of-00032.safetensors'
|
| 25 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00013-of-00032.safetensors'
|
| 26 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00014-of-00032.safetensors'
|
| 27 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00015-of-00032.safetensors'
|
| 28 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00016-of-00032.safetensors'
|
| 29 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00017-of-00032.safetensors'
|
| 30 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00018-of-00032.safetensors'
|
| 31 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00019-of-00032.safetensors'
|
| 32 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00020-of-00032.safetensors'
|
| 33 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00021-of-00032.safetensors'
|
| 34 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00022-of-00032.safetensors'
|
| 35 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00023-of-00032.safetensors'
|
| 36 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00024-of-00032.safetensors'
|
| 37 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00025-of-00032.safetensors'
|
| 38 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00026-of-00032.safetensors'
|
| 39 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00027-of-00032.safetensors'
|
| 40 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00028-of-00032.safetensors'
|
| 41 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00029-of-00032.safetensors'
|
| 42 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00030-of-00032.safetensors'
|
| 43 |
+
INFO:hf-to-gguf:gguf: indexing model part 'model-00031-of-00032.safetensors'
|
| 44 |
+
INFO:gguf.gguf_writer:gguf: This GGUF file is for Little Endian only
|
| 45 |
+
INFO:hf-to-gguf:Exporting model...
|
| 46 |
+
INFO:hf-to-gguf:blk.0.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 47 |
+
INFO:hf-to-gguf:blk.0.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 48 |
+
INFO:hf-to-gguf:blk.0.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 49 |
+
INFO:hf-to-gguf:blk.0.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 50 |
+
INFO:hf-to-gguf:blk.0.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 51 |
+
INFO:hf-to-gguf:blk.0.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 52 |
+
INFO:hf-to-gguf:blk.0.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 53 |
+
INFO:hf-to-gguf:blk.0.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 54 |
+
INFO:hf-to-gguf:blk.0.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 55 |
+
INFO:hf-to-gguf:blk.0.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 56 |
+
INFO:hf-to-gguf:blk.0.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 57 |
+
INFO:hf-to-gguf:blk.0.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 58 |
+
INFO:hf-to-gguf:blk.0.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 59 |
+
INFO:hf-to-gguf:blk.0.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 60 |
+
INFO:hf-to-gguf:blk.0.ffn_down.weight, torch.bfloat16 --> BF16, shape = {4608, 1536}
|
| 61 |
+
INFO:hf-to-gguf:blk.0.ffn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 4608}
|
| 62 |
+
INFO:hf-to-gguf:blk.0.ffn_up.weight, torch.bfloat16 --> BF16, shape = {1536, 4608}
|
| 63 |
+
INFO:hf-to-gguf:blk.0.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 64 |
+
INFO:hf-to-gguf:blk.1.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 65 |
+
INFO:hf-to-gguf:blk.1.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 66 |
+
INFO:hf-to-gguf:blk.1.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 67 |
+
INFO:hf-to-gguf:blk.1.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 68 |
+
INFO:hf-to-gguf:blk.1.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 69 |
+
INFO:hf-to-gguf:blk.1.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 70 |
+
INFO:hf-to-gguf:blk.1.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 71 |
+
INFO:hf-to-gguf:blk.1.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 72 |
+
INFO:hf-to-gguf:blk.1.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 73 |
+
INFO:hf-to-gguf:blk.1.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 74 |
+
INFO:hf-to-gguf:blk.1.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 75 |
+
INFO:hf-to-gguf:blk.1.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 76 |
+
INFO:hf-to-gguf:blk.1.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 77 |
+
INFO:hf-to-gguf:blk.1.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 78 |
+
INFO:hf-to-gguf:blk.1.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 79 |
+
INFO:hf-to-gguf:blk.1.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 80 |
+
INFO:hf-to-gguf:blk.1.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 81 |
+
INFO:hf-to-gguf:blk.1.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 82 |
+
INFO:hf-to-gguf:blk.1.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 83 |
+
INFO:hf-to-gguf:blk.1.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 84 |
+
INFO:hf-to-gguf:blk.2.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 85 |
+
INFO:hf-to-gguf:blk.2.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 86 |
+
INFO:hf-to-gguf:blk.2.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 87 |
+
INFO:hf-to-gguf:blk.2.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 88 |
+
INFO:hf-to-gguf:blk.2.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 89 |
+
INFO:hf-to-gguf:blk.2.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 90 |
+
INFO:hf-to-gguf:blk.2.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 91 |
+
INFO:hf-to-gguf:blk.2.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 92 |
+
INFO:hf-to-gguf:blk.2.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 93 |
+
INFO:hf-to-gguf:blk.2.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 94 |
+
INFO:hf-to-gguf:blk.2.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 95 |
+
INFO:hf-to-gguf:blk.2.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 96 |
+
INFO:hf-to-gguf:blk.2.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 97 |
+
INFO:hf-to-gguf:blk.2.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 98 |
+
INFO:hf-to-gguf:blk.2.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 99 |
+
INFO:hf-to-gguf:blk.2.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 100 |
+
INFO:hf-to-gguf:blk.2.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 101 |
+
INFO:hf-to-gguf:blk.2.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 102 |
+
INFO:hf-to-gguf:blk.2.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 103 |
+
INFO:hf-to-gguf:blk.2.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 104 |
+
INFO:hf-to-gguf:blk.3.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 105 |
+
INFO:hf-to-gguf:blk.3.attn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 106 |
+
INFO:hf-to-gguf:blk.3.attn_kv_a_norm.weight, torch.bfloat16 --> F32, shape = {512}
|
| 107 |
+
INFO:hf-to-gguf:blk.3.attn_kv_a_mqa.weight, torch.bfloat16 --> BF16, shape = {1536, 576}
|
| 108 |
+
INFO:hf-to-gguf:blk.3.attn_k_b.weight, torch.bfloat16 --> BF16, shape = {128, 512, 16}
|
| 109 |
+
INFO:hf-to-gguf:blk.3.attn_v_b.weight, torch.bfloat16 --> BF16, shape = {512, 128, 16}
|
| 110 |
+
INFO:hf-to-gguf:blk.3.attn_q_a_norm.weight, torch.bfloat16 --> F32, shape = {256}
|
| 111 |
+
INFO:hf-to-gguf:blk.3.attn_q_a.weight, torch.bfloat16 --> BF16, shape = {1536, 256}
|
| 112 |
+
INFO:hf-to-gguf:blk.3.attn_q_b.weight, torch.bfloat16 --> BF16, shape = {256, 3072}
|
| 113 |
+
INFO:hf-to-gguf:blk.3.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 114 |
+
INFO:hf-to-gguf:blk.3.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 115 |
+
INFO:hf-to-gguf:blk.3.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 116 |
+
INFO:hf-to-gguf:blk.3.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 117 |
+
INFO:hf-to-gguf:blk.3.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 118 |
+
INFO:hf-to-gguf:blk.3.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 119 |
+
INFO:hf-to-gguf:blk.3.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 120 |
+
INFO:hf-to-gguf:blk.4.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 121 |
+
INFO:hf-to-gguf:blk.4.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 122 |
+
INFO:hf-to-gguf:blk.4.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 123 |
+
INFO:hf-to-gguf:blk.4.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 124 |
+
INFO:hf-to-gguf:blk.4.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 125 |
+
INFO:hf-to-gguf:blk.4.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 126 |
+
INFO:hf-to-gguf:blk.4.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 127 |
+
INFO:hf-to-gguf:blk.4.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 128 |
+
INFO:hf-to-gguf:blk.4.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 129 |
+
INFO:hf-to-gguf:blk.4.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 130 |
+
INFO:hf-to-gguf:blk.4.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 131 |
+
INFO:hf-to-gguf:blk.4.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 132 |
+
INFO:hf-to-gguf:blk.4.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 133 |
+
INFO:hf-to-gguf:blk.4.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 134 |
+
INFO:hf-to-gguf:blk.4.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 135 |
+
INFO:hf-to-gguf:blk.4.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 136 |
+
INFO:hf-to-gguf:blk.4.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 137 |
+
INFO:hf-to-gguf:blk.4.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 138 |
+
INFO:hf-to-gguf:blk.4.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 139 |
+
INFO:hf-to-gguf:blk.4.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 140 |
+
INFO:hf-to-gguf:blk.5.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 141 |
+
INFO:hf-to-gguf:blk.5.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 142 |
+
INFO:hf-to-gguf:blk.5.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 143 |
+
INFO:hf-to-gguf:blk.5.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 144 |
+
INFO:hf-to-gguf:blk.5.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 145 |
+
INFO:hf-to-gguf:blk.5.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 146 |
+
INFO:hf-to-gguf:blk.5.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 147 |
+
INFO:hf-to-gguf:blk.5.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 148 |
+
INFO:hf-to-gguf:blk.5.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 149 |
+
INFO:hf-to-gguf:blk.5.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 150 |
+
INFO:hf-to-gguf:blk.5.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 151 |
+
INFO:hf-to-gguf:blk.5.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 152 |
+
INFO:hf-to-gguf:blk.5.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 153 |
+
INFO:hf-to-gguf:blk.5.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 154 |
+
INFO:hf-to-gguf:blk.5.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 155 |
+
INFO:hf-to-gguf:blk.5.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 156 |
+
INFO:hf-to-gguf:blk.5.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 157 |
+
INFO:hf-to-gguf:blk.5.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 158 |
+
INFO:hf-to-gguf:blk.5.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 159 |
+
INFO:hf-to-gguf:blk.5.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 160 |
+
INFO:hf-to-gguf:token_embd.weight, torch.bfloat16 --> BF16, shape = {1536, 157184}
|
| 161 |
+
INFO:hf-to-gguf:blk.1.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 162 |
+
INFO:hf-to-gguf:blk.1.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 163 |
+
INFO:hf-to-gguf:blk.1.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 164 |
+
INFO:hf-to-gguf:blk.2.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 165 |
+
INFO:hf-to-gguf:blk.2.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 166 |
+
INFO:hf-to-gguf:blk.2.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 167 |
+
INFO:hf-to-gguf:blk.3.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 168 |
+
INFO:hf-to-gguf:blk.3.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 169 |
+
INFO:hf-to-gguf:blk.3.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 170 |
+
INFO:hf-to-gguf:blk.4.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 171 |
+
INFO:hf-to-gguf:blk.4.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 172 |
+
INFO:hf-to-gguf:blk.4.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 173 |
+
INFO:hf-to-gguf:blk.5.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 174 |
+
INFO:hf-to-gguf:blk.5.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 175 |
+
INFO:hf-to-gguf:blk.5.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 176 |
+
INFO:hf-to-gguf:blk.10.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 177 |
+
INFO:hf-to-gguf:blk.10.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 178 |
+
INFO:hf-to-gguf:blk.10.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 179 |
+
INFO:hf-to-gguf:blk.10.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 180 |
+
INFO:hf-to-gguf:blk.10.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 181 |
+
INFO:hf-to-gguf:blk.10.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 182 |
+
INFO:hf-to-gguf:blk.10.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 183 |
+
INFO:hf-to-gguf:blk.10.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 184 |
+
INFO:hf-to-gguf:blk.10.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 185 |
+
INFO:hf-to-gguf:blk.10.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 186 |
+
INFO:hf-to-gguf:blk.10.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 187 |
+
INFO:hf-to-gguf:blk.10.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 188 |
+
INFO:hf-to-gguf:blk.10.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 189 |
+
INFO:hf-to-gguf:blk.10.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 190 |
+
INFO:hf-to-gguf:blk.10.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 191 |
+
INFO:hf-to-gguf:blk.10.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 192 |
+
INFO:hf-to-gguf:blk.10.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 193 |
+
INFO:hf-to-gguf:blk.10.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 194 |
+
INFO:hf-to-gguf:blk.10.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 195 |
+
INFO:hf-to-gguf:blk.10.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 196 |
+
INFO:hf-to-gguf:blk.11.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 197 |
+
INFO:hf-to-gguf:blk.11.attn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 198 |
+
INFO:hf-to-gguf:blk.11.attn_kv_a_norm.weight, torch.bfloat16 --> F32, shape = {512}
|
| 199 |
+
INFO:hf-to-gguf:blk.11.attn_kv_a_mqa.weight, torch.bfloat16 --> BF16, shape = {1536, 576}
|
| 200 |
+
INFO:hf-to-gguf:blk.11.attn_k_b.weight, torch.bfloat16 --> BF16, shape = {128, 512, 16}
|
| 201 |
+
INFO:hf-to-gguf:blk.11.attn_v_b.weight, torch.bfloat16 --> BF16, shape = {512, 128, 16}
|
| 202 |
+
INFO:hf-to-gguf:blk.11.attn_q_a_norm.weight, torch.bfloat16 --> F32, shape = {256}
|
| 203 |
+
INFO:hf-to-gguf:blk.11.attn_q_a.weight, torch.bfloat16 --> BF16, shape = {1536, 256}
|
| 204 |
+
INFO:hf-to-gguf:blk.11.attn_q_b.weight, torch.bfloat16 --> BF16, shape = {256, 3072}
|
| 205 |
+
INFO:hf-to-gguf:blk.11.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 206 |
+
INFO:hf-to-gguf:blk.11.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 207 |
+
INFO:hf-to-gguf:blk.11.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 208 |
+
INFO:hf-to-gguf:blk.11.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 209 |
+
INFO:hf-to-gguf:blk.11.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 210 |
+
INFO:hf-to-gguf:blk.11.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 211 |
+
INFO:hf-to-gguf:blk.11.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 212 |
+
INFO:hf-to-gguf:blk.6.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 213 |
+
INFO:hf-to-gguf:blk.6.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 214 |
+
INFO:hf-to-gguf:blk.6.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 215 |
+
INFO:hf-to-gguf:blk.6.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 216 |
+
INFO:hf-to-gguf:blk.6.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 217 |
+
INFO:hf-to-gguf:blk.6.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 218 |
+
INFO:hf-to-gguf:blk.6.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 219 |
+
INFO:hf-to-gguf:blk.6.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 220 |
+
INFO:hf-to-gguf:blk.6.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 221 |
+
INFO:hf-to-gguf:blk.6.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 222 |
+
INFO:hf-to-gguf:blk.6.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 223 |
+
INFO:hf-to-gguf:blk.6.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 224 |
+
INFO:hf-to-gguf:blk.6.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 225 |
+
INFO:hf-to-gguf:blk.6.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 226 |
+
INFO:hf-to-gguf:blk.6.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 227 |
+
INFO:hf-to-gguf:blk.6.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 228 |
+
INFO:hf-to-gguf:blk.6.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 229 |
+
INFO:hf-to-gguf:blk.6.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 230 |
+
INFO:hf-to-gguf:blk.6.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 231 |
+
INFO:hf-to-gguf:blk.6.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 232 |
+
INFO:hf-to-gguf:blk.7.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 233 |
+
INFO:hf-to-gguf:blk.7.attn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 234 |
+
INFO:hf-to-gguf:blk.7.attn_kv_a_norm.weight, torch.bfloat16 --> F32, shape = {512}
|
| 235 |
+
INFO:hf-to-gguf:blk.7.attn_kv_a_mqa.weight, torch.bfloat16 --> BF16, shape = {1536, 576}
|
| 236 |
+
INFO:hf-to-gguf:blk.7.attn_k_b.weight, torch.bfloat16 --> BF16, shape = {128, 512, 16}
|
| 237 |
+
INFO:hf-to-gguf:blk.7.attn_v_b.weight, torch.bfloat16 --> BF16, shape = {512, 128, 16}
|
| 238 |
+
INFO:hf-to-gguf:blk.7.attn_q_a_norm.weight, torch.bfloat16 --> F32, shape = {256}
|
| 239 |
+
INFO:hf-to-gguf:blk.7.attn_q_a.weight, torch.bfloat16 --> BF16, shape = {1536, 256}
|
| 240 |
+
INFO:hf-to-gguf:blk.7.attn_q_b.weight, torch.bfloat16 --> BF16, shape = {256, 3072}
|
| 241 |
+
INFO:hf-to-gguf:blk.7.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 242 |
+
INFO:hf-to-gguf:blk.7.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 243 |
+
INFO:hf-to-gguf:blk.7.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 244 |
+
INFO:hf-to-gguf:blk.7.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 245 |
+
INFO:hf-to-gguf:blk.7.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 246 |
+
INFO:hf-to-gguf:blk.7.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 247 |
+
INFO:hf-to-gguf:blk.7.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 248 |
+
INFO:hf-to-gguf:blk.8.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 249 |
+
INFO:hf-to-gguf:blk.8.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 250 |
+
INFO:hf-to-gguf:blk.8.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 251 |
+
INFO:hf-to-gguf:blk.8.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 252 |
+
INFO:hf-to-gguf:blk.8.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 253 |
+
INFO:hf-to-gguf:blk.8.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 254 |
+
INFO:hf-to-gguf:blk.8.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 255 |
+
INFO:hf-to-gguf:blk.8.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 256 |
+
INFO:hf-to-gguf:blk.8.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 257 |
+
INFO:hf-to-gguf:blk.8.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 258 |
+
INFO:hf-to-gguf:blk.8.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 259 |
+
INFO:hf-to-gguf:blk.8.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 260 |
+
INFO:hf-to-gguf:blk.8.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 261 |
+
INFO:hf-to-gguf:blk.8.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 262 |
+
INFO:hf-to-gguf:blk.8.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 263 |
+
INFO:hf-to-gguf:blk.8.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 264 |
+
INFO:hf-to-gguf:blk.8.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 265 |
+
INFO:hf-to-gguf:blk.8.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 266 |
+
INFO:hf-to-gguf:blk.8.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 267 |
+
INFO:hf-to-gguf:blk.8.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 268 |
+
INFO:hf-to-gguf:blk.9.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 269 |
+
INFO:hf-to-gguf:blk.9.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 270 |
+
INFO:hf-to-gguf:blk.9.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 271 |
+
INFO:hf-to-gguf:blk.9.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 272 |
+
INFO:hf-to-gguf:blk.9.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 273 |
+
INFO:hf-to-gguf:blk.9.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 274 |
+
INFO:hf-to-gguf:blk.9.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 275 |
+
INFO:hf-to-gguf:blk.9.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 276 |
+
INFO:hf-to-gguf:blk.9.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 277 |
+
INFO:hf-to-gguf:blk.9.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 278 |
+
INFO:hf-to-gguf:blk.9.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 279 |
+
INFO:hf-to-gguf:blk.9.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 280 |
+
INFO:hf-to-gguf:blk.9.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 281 |
+
INFO:hf-to-gguf:blk.9.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 282 |
+
INFO:hf-to-gguf:blk.9.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 283 |
+
INFO:hf-to-gguf:blk.9.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 284 |
+
INFO:hf-to-gguf:blk.9.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 285 |
+
INFO:hf-to-gguf:blk.9.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 286 |
+
INFO:hf-to-gguf:blk.9.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 287 |
+
INFO:hf-to-gguf:blk.9.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 288 |
+
INFO:hf-to-gguf:blk.10.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 289 |
+
INFO:hf-to-gguf:blk.10.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 290 |
+
INFO:hf-to-gguf:blk.10.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 291 |
+
INFO:hf-to-gguf:blk.11.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 292 |
+
INFO:hf-to-gguf:blk.11.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 293 |
+
INFO:hf-to-gguf:blk.11.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 294 |
+
INFO:hf-to-gguf:blk.6.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 295 |
+
INFO:hf-to-gguf:blk.6.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 296 |
+
INFO:hf-to-gguf:blk.6.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 297 |
+
INFO:hf-to-gguf:blk.7.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 298 |
+
INFO:hf-to-gguf:blk.7.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 299 |
+
INFO:hf-to-gguf:blk.7.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 300 |
+
INFO:hf-to-gguf:blk.8.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 301 |
+
INFO:hf-to-gguf:blk.8.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 302 |
+
INFO:hf-to-gguf:blk.8.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 303 |
+
INFO:hf-to-gguf:blk.9.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 304 |
+
INFO:hf-to-gguf:blk.9.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 305 |
+
INFO:hf-to-gguf:blk.9.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 306 |
+
INFO:hf-to-gguf:blk.12.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 307 |
+
INFO:hf-to-gguf:blk.12.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 308 |
+
INFO:hf-to-gguf:blk.12.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 309 |
+
INFO:hf-to-gguf:blk.12.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 310 |
+
INFO:hf-to-gguf:blk.12.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 311 |
+
INFO:hf-to-gguf:blk.12.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 312 |
+
INFO:hf-to-gguf:blk.12.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 313 |
+
INFO:hf-to-gguf:blk.12.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 314 |
+
INFO:hf-to-gguf:blk.12.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 315 |
+
INFO:hf-to-gguf:blk.12.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 316 |
+
INFO:hf-to-gguf:blk.12.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 317 |
+
INFO:hf-to-gguf:blk.12.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 318 |
+
INFO:hf-to-gguf:blk.12.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 319 |
+
INFO:hf-to-gguf:blk.12.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 320 |
+
INFO:hf-to-gguf:blk.12.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 321 |
+
INFO:hf-to-gguf:blk.12.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 322 |
+
INFO:hf-to-gguf:blk.12.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 323 |
+
INFO:hf-to-gguf:blk.12.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 324 |
+
INFO:hf-to-gguf:blk.12.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 325 |
+
INFO:hf-to-gguf:blk.12.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 326 |
+
INFO:hf-to-gguf:blk.13.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 327 |
+
INFO:hf-to-gguf:blk.13.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 328 |
+
INFO:hf-to-gguf:blk.13.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 329 |
+
INFO:hf-to-gguf:blk.13.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 330 |
+
INFO:hf-to-gguf:blk.13.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 331 |
+
INFO:hf-to-gguf:blk.13.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 332 |
+
INFO:hf-to-gguf:blk.13.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 333 |
+
INFO:hf-to-gguf:blk.13.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 334 |
+
INFO:hf-to-gguf:blk.13.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 335 |
+
INFO:hf-to-gguf:blk.13.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 336 |
+
INFO:hf-to-gguf:blk.13.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 337 |
+
INFO:hf-to-gguf:blk.13.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 338 |
+
INFO:hf-to-gguf:blk.13.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 339 |
+
INFO:hf-to-gguf:blk.13.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 340 |
+
INFO:hf-to-gguf:blk.13.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 341 |
+
INFO:hf-to-gguf:blk.13.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 342 |
+
INFO:hf-to-gguf:blk.13.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 343 |
+
INFO:hf-to-gguf:blk.13.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 344 |
+
INFO:hf-to-gguf:blk.13.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 345 |
+
INFO:hf-to-gguf:blk.13.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 346 |
+
INFO:hf-to-gguf:blk.14.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 347 |
+
INFO:hf-to-gguf:blk.14.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 348 |
+
INFO:hf-to-gguf:blk.14.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 349 |
+
INFO:hf-to-gguf:blk.14.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 350 |
+
INFO:hf-to-gguf:blk.14.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 351 |
+
INFO:hf-to-gguf:blk.14.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 352 |
+
INFO:hf-to-gguf:blk.14.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 353 |
+
INFO:hf-to-gguf:blk.14.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 354 |
+
INFO:hf-to-gguf:blk.14.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 355 |
+
INFO:hf-to-gguf:blk.14.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 356 |
+
INFO:hf-to-gguf:blk.14.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 357 |
+
INFO:hf-to-gguf:blk.14.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 358 |
+
INFO:hf-to-gguf:blk.14.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 359 |
+
INFO:hf-to-gguf:blk.14.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 360 |
+
INFO:hf-to-gguf:blk.14.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 361 |
+
INFO:hf-to-gguf:blk.14.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 362 |
+
INFO:hf-to-gguf:blk.14.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 363 |
+
INFO:hf-to-gguf:blk.14.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 364 |
+
INFO:hf-to-gguf:blk.14.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 365 |
+
INFO:hf-to-gguf:blk.14.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 366 |
+
INFO:hf-to-gguf:blk.15.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 367 |
+
INFO:hf-to-gguf:blk.15.attn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 368 |
+
INFO:hf-to-gguf:blk.15.attn_kv_a_norm.weight, torch.bfloat16 --> F32, shape = {512}
|
| 369 |
+
INFO:hf-to-gguf:blk.15.attn_kv_a_mqa.weight, torch.bfloat16 --> BF16, shape = {1536, 576}
|
| 370 |
+
INFO:hf-to-gguf:blk.15.attn_k_b.weight, torch.bfloat16 --> BF16, shape = {128, 512, 16}
|
| 371 |
+
INFO:hf-to-gguf:blk.15.attn_v_b.weight, torch.bfloat16 --> BF16, shape = {512, 128, 16}
|
| 372 |
+
INFO:hf-to-gguf:blk.15.attn_q_a_norm.weight, torch.bfloat16 --> F32, shape = {256}
|
| 373 |
+
INFO:hf-to-gguf:blk.15.attn_q_a.weight, torch.bfloat16 --> BF16, shape = {1536, 256}
|
| 374 |
+
INFO:hf-to-gguf:blk.15.attn_q_b.weight, torch.bfloat16 --> BF16, shape = {256, 3072}
|
| 375 |
+
INFO:hf-to-gguf:blk.15.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 376 |
+
INFO:hf-to-gguf:blk.15.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 377 |
+
INFO:hf-to-gguf:blk.15.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 378 |
+
INFO:hf-to-gguf:blk.15.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 379 |
+
INFO:hf-to-gguf:blk.15.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 380 |
+
INFO:hf-to-gguf:blk.15.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 381 |
+
INFO:hf-to-gguf:blk.15.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 382 |
+
INFO:hf-to-gguf:blk.16.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 383 |
+
INFO:hf-to-gguf:blk.16.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 384 |
+
INFO:hf-to-gguf:blk.16.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 385 |
+
INFO:hf-to-gguf:blk.16.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 386 |
+
INFO:hf-to-gguf:blk.16.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 387 |
+
INFO:hf-to-gguf:blk.16.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 388 |
+
INFO:hf-to-gguf:blk.16.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 389 |
+
INFO:hf-to-gguf:blk.16.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 390 |
+
INFO:hf-to-gguf:blk.16.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 391 |
+
INFO:hf-to-gguf:blk.16.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 392 |
+
INFO:hf-to-gguf:blk.16.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 393 |
+
INFO:hf-to-gguf:blk.16.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 394 |
+
INFO:hf-to-gguf:blk.16.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 395 |
+
INFO:hf-to-gguf:blk.16.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 396 |
+
INFO:hf-to-gguf:blk.16.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 397 |
+
INFO:hf-to-gguf:blk.16.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 398 |
+
INFO:hf-to-gguf:blk.16.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 399 |
+
INFO:hf-to-gguf:blk.16.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 400 |
+
INFO:hf-to-gguf:blk.16.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 401 |
+
INFO:hf-to-gguf:blk.16.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 402 |
+
INFO:hf-to-gguf:blk.17.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 403 |
+
INFO:hf-to-gguf:blk.17.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 404 |
+
INFO:hf-to-gguf:blk.17.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 405 |
+
INFO:hf-to-gguf:blk.17.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 406 |
+
INFO:hf-to-gguf:blk.17.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 407 |
+
INFO:hf-to-gguf:blk.17.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 408 |
+
INFO:hf-to-gguf:blk.17.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 409 |
+
INFO:hf-to-gguf:blk.17.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 410 |
+
INFO:hf-to-gguf:blk.17.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 411 |
+
INFO:hf-to-gguf:blk.17.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 412 |
+
INFO:hf-to-gguf:blk.17.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 413 |
+
INFO:hf-to-gguf:blk.17.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 414 |
+
INFO:hf-to-gguf:blk.17.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 415 |
+
INFO:hf-to-gguf:blk.17.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 416 |
+
INFO:hf-to-gguf:blk.17.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 417 |
+
INFO:hf-to-gguf:blk.17.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 418 |
+
INFO:hf-to-gguf:blk.17.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 419 |
+
INFO:hf-to-gguf:blk.17.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 420 |
+
INFO:hf-to-gguf:blk.17.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 421 |
+
INFO:hf-to-gguf:blk.17.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 422 |
+
INFO:hf-to-gguf:blk.12.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 423 |
+
INFO:hf-to-gguf:blk.12.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 424 |
+
INFO:hf-to-gguf:blk.12.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 425 |
+
INFO:hf-to-gguf:blk.13.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 426 |
+
INFO:hf-to-gguf:blk.13.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 427 |
+
INFO:hf-to-gguf:blk.13.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 428 |
+
INFO:hf-to-gguf:blk.14.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 429 |
+
INFO:hf-to-gguf:blk.14.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 430 |
+
INFO:hf-to-gguf:blk.14.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 431 |
+
INFO:hf-to-gguf:blk.15.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 432 |
+
INFO:hf-to-gguf:blk.15.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 433 |
+
INFO:hf-to-gguf:blk.15.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 434 |
+
INFO:hf-to-gguf:blk.16.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 435 |
+
INFO:hf-to-gguf:blk.16.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 436 |
+
INFO:hf-to-gguf:blk.16.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 437 |
+
INFO:hf-to-gguf:blk.17.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 438 |
+
INFO:hf-to-gguf:blk.17.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 439 |
+
INFO:hf-to-gguf:blk.17.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 440 |
+
INFO:hf-to-gguf:output.weight, torch.bfloat16 --> BF16, shape = {1536, 157184}
|
| 441 |
+
INFO:hf-to-gguf:blk.18.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 442 |
+
INFO:hf-to-gguf:blk.18.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 443 |
+
INFO:hf-to-gguf:blk.18.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 444 |
+
INFO:hf-to-gguf:blk.18.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 445 |
+
INFO:hf-to-gguf:blk.18.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 446 |
+
INFO:hf-to-gguf:blk.18.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 447 |
+
INFO:hf-to-gguf:blk.18.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 448 |
+
INFO:hf-to-gguf:blk.18.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 449 |
+
INFO:hf-to-gguf:blk.18.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 450 |
+
INFO:hf-to-gguf:blk.18.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 451 |
+
INFO:hf-to-gguf:blk.18.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 452 |
+
INFO:hf-to-gguf:blk.18.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 453 |
+
INFO:hf-to-gguf:blk.18.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 454 |
+
INFO:hf-to-gguf:blk.18.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 455 |
+
INFO:hf-to-gguf:blk.18.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 456 |
+
INFO:hf-to-gguf:blk.18.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 457 |
+
INFO:hf-to-gguf:blk.18.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 458 |
+
INFO:hf-to-gguf:blk.18.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 459 |
+
INFO:hf-to-gguf:blk.18.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 460 |
+
INFO:hf-to-gguf:blk.18.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 461 |
+
INFO:hf-to-gguf:blk.19.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 462 |
+
INFO:hf-to-gguf:blk.19.attn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 463 |
+
INFO:hf-to-gguf:blk.19.attn_kv_a_norm.weight, torch.bfloat16 --> F32, shape = {512}
|
| 464 |
+
INFO:hf-to-gguf:blk.19.attn_kv_a_mqa.weight, torch.bfloat16 --> BF16, shape = {1536, 576}
|
| 465 |
+
INFO:hf-to-gguf:blk.19.attn_k_b.weight, torch.bfloat16 --> BF16, shape = {128, 512, 16}
|
| 466 |
+
INFO:hf-to-gguf:blk.19.attn_v_b.weight, torch.bfloat16 --> BF16, shape = {512, 128, 16}
|
| 467 |
+
INFO:hf-to-gguf:blk.19.attn_q_a_norm.weight, torch.bfloat16 --> F32, shape = {256}
|
| 468 |
+
INFO:hf-to-gguf:blk.19.attn_q_a.weight, torch.bfloat16 --> BF16, shape = {1536, 256}
|
| 469 |
+
INFO:hf-to-gguf:blk.19.attn_q_b.weight, torch.bfloat16 --> BF16, shape = {256, 3072}
|
| 470 |
+
INFO:hf-to-gguf:blk.19.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 471 |
+
INFO:hf-to-gguf:blk.19.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 472 |
+
INFO:hf-to-gguf:blk.19.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 473 |
+
INFO:hf-to-gguf:blk.19.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 474 |
+
INFO:hf-to-gguf:blk.19.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 475 |
+
INFO:hf-to-gguf:blk.19.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 476 |
+
INFO:hf-to-gguf:blk.19.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 477 |
+
INFO:hf-to-gguf:blk.20.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 478 |
+
INFO:hf-to-gguf:blk.20.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 479 |
+
INFO:hf-to-gguf:blk.20.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 480 |
+
INFO:hf-to-gguf:blk.20.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 481 |
+
INFO:hf-to-gguf:blk.20.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 482 |
+
INFO:hf-to-gguf:blk.20.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 483 |
+
INFO:hf-to-gguf:blk.20.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 484 |
+
INFO:hf-to-gguf:blk.20.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 485 |
+
INFO:hf-to-gguf:blk.20.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 486 |
+
INFO:hf-to-gguf:blk.20.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 487 |
+
INFO:hf-to-gguf:blk.20.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 488 |
+
INFO:hf-to-gguf:blk.20.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 489 |
+
INFO:hf-to-gguf:blk.20.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 490 |
+
INFO:hf-to-gguf:blk.20.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 491 |
+
INFO:hf-to-gguf:blk.20.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 492 |
+
INFO:hf-to-gguf:blk.20.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 493 |
+
INFO:hf-to-gguf:blk.20.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 494 |
+
INFO:hf-to-gguf:blk.20.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 495 |
+
INFO:hf-to-gguf:blk.20.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 496 |
+
INFO:hf-to-gguf:blk.20.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 497 |
+
INFO:hf-to-gguf:blk.21.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 498 |
+
INFO:hf-to-gguf:blk.21.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 499 |
+
INFO:hf-to-gguf:blk.21.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 500 |
+
INFO:hf-to-gguf:blk.21.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 501 |
+
INFO:hf-to-gguf:blk.21.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 502 |
+
INFO:hf-to-gguf:blk.21.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 503 |
+
INFO:hf-to-gguf:blk.21.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 504 |
+
INFO:hf-to-gguf:blk.21.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 505 |
+
INFO:hf-to-gguf:blk.21.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 506 |
+
INFO:hf-to-gguf:blk.21.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 507 |
+
INFO:hf-to-gguf:blk.21.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 508 |
+
INFO:hf-to-gguf:blk.21.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 509 |
+
INFO:hf-to-gguf:blk.21.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 510 |
+
INFO:hf-to-gguf:blk.21.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 511 |
+
INFO:hf-to-gguf:blk.21.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 512 |
+
INFO:hf-to-gguf:blk.21.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 513 |
+
INFO:hf-to-gguf:blk.21.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 514 |
+
INFO:hf-to-gguf:blk.21.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 515 |
+
INFO:hf-to-gguf:blk.21.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 516 |
+
INFO:hf-to-gguf:blk.21.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 517 |
+
INFO:hf-to-gguf:blk.22.ssm_a, torch.float32 --> F32, shape = {1, 16}
|
| 518 |
+
INFO:hf-to-gguf:blk.22.ssm_beta.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 519 |
+
INFO:hf-to-gguf:blk.22.ssm_dt.bias, torch.float32 --> F32, shape = {2048}
|
| 520 |
+
INFO:hf-to-gguf:blk.22.ssm_f_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 521 |
+
INFO:hf-to-gguf:blk.22.ssm_g_a.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 522 |
+
INFO:hf-to-gguf:blk.22.ssm_conv1d_k.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 523 |
+
INFO:hf-to-gguf:blk.22.attn_k.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 524 |
+
INFO:hf-to-gguf:blk.22.ssm_norm.weight, torch.bfloat16 --> F32, shape = {128}
|
| 525 |
+
INFO:hf-to-gguf:blk.22.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 526 |
+
INFO:hf-to-gguf:blk.22.ssm_conv1d_q.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 527 |
+
INFO:hf-to-gguf:blk.22.attn_q.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 528 |
+
INFO:hf-to-gguf:blk.22.ssm_conv1d_v.weight, torch.bfloat16 --> F32, shape = {4, 1, 2048, 1}
|
| 529 |
+
INFO:hf-to-gguf:blk.22.attn_v.weight, torch.bfloat16 --> BF16, shape = {1536, 2048}
|
| 530 |
+
INFO:hf-to-gguf:blk.22.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 531 |
+
INFO:hf-to-gguf:blk.22.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 532 |
+
INFO:hf-to-gguf:blk.22.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 533 |
+
INFO:hf-to-gguf:blk.22.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 534 |
+
INFO:hf-to-gguf:blk.22.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 535 |
+
INFO:hf-to-gguf:blk.22.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 536 |
+
INFO:hf-to-gguf:blk.22.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 537 |
+
INFO:hf-to-gguf:blk.23.attn_output.weight, torch.bfloat16 --> BF16, shape = {2048, 1536}
|
| 538 |
+
INFO:hf-to-gguf:blk.23.attn_gate.weight, torch.bfloat16 --> BF16, shape = {1536, 16}
|
| 539 |
+
INFO:hf-to-gguf:blk.23.attn_kv_a_norm.weight, torch.bfloat16 --> F32, shape = {512}
|
| 540 |
+
INFO:hf-to-gguf:blk.23.attn_kv_a_mqa.weight, torch.bfloat16 --> BF16, shape = {1536, 576}
|
| 541 |
+
INFO:hf-to-gguf:blk.23.attn_k_b.weight, torch.bfloat16 --> BF16, shape = {128, 512, 16}
|
| 542 |
+
INFO:hf-to-gguf:blk.23.attn_v_b.weight, torch.bfloat16 --> BF16, shape = {512, 128, 16}
|
| 543 |
+
INFO:hf-to-gguf:blk.23.attn_q_a_norm.weight, torch.bfloat16 --> F32, shape = {256}
|
| 544 |
+
INFO:hf-to-gguf:blk.23.attn_q_a.weight, torch.bfloat16 --> BF16, shape = {1536, 256}
|
| 545 |
+
INFO:hf-to-gguf:blk.23.attn_q_b.weight, torch.bfloat16 --> BF16, shape = {256, 3072}
|
| 546 |
+
INFO:hf-to-gguf:blk.23.attn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 547 |
+
INFO:hf-to-gguf:blk.23.exp_probs_b.bias, torch.float32 --> F32, shape = {128}
|
| 548 |
+
INFO:hf-to-gguf:blk.23.ffn_gate_inp.weight, torch.bfloat16 --> F32, shape = {1536, 128}
|
| 549 |
+
INFO:hf-to-gguf:blk.23.ffn_down_shexp.weight, torch.bfloat16 --> BF16, shape = {512, 1536}
|
| 550 |
+
INFO:hf-to-gguf:blk.23.ffn_gate_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 551 |
+
INFO:hf-to-gguf:blk.23.ffn_up_shexp.weight, torch.bfloat16 --> BF16, shape = {1536, 512}
|
| 552 |
+
INFO:hf-to-gguf:blk.23.ffn_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 553 |
+
INFO:hf-to-gguf:output_norm.weight, torch.bfloat16 --> F32, shape = {1536}
|
| 554 |
+
INFO:hf-to-gguf:blk.18.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 555 |
+
INFO:hf-to-gguf:blk.18.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 556 |
+
INFO:hf-to-gguf:blk.18.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 557 |
+
INFO:hf-to-gguf:blk.19.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 558 |
+
INFO:hf-to-gguf:blk.19.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 559 |
+
INFO:hf-to-gguf:blk.19.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 560 |
+
INFO:hf-to-gguf:blk.20.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 561 |
+
INFO:hf-to-gguf:blk.20.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 562 |
+
INFO:hf-to-gguf:blk.20.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 563 |
+
INFO:hf-to-gguf:blk.21.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 564 |
+
INFO:hf-to-gguf:blk.21.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 565 |
+
INFO:hf-to-gguf:blk.21.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 566 |
+
INFO:hf-to-gguf:blk.22.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 567 |
+
INFO:hf-to-gguf:blk.22.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 568 |
+
INFO:hf-to-gguf:blk.22.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 569 |
+
INFO:hf-to-gguf:blk.23.ffn_down_exps.weight, torch.bfloat16 --> BF16, shape = {512, 1536, 128}
|
| 570 |
+
INFO:hf-to-gguf:blk.23.ffn_gate_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 571 |
+
INFO:hf-to-gguf:blk.23.ffn_up_exps.weight, torch.bfloat16 --> BF16, shape = {1536, 512, 128}
|
| 572 |
+
INFO:hf-to-gguf:Set meta model
|
| 573 |
+
INFO:hf-to-gguf:Set model parameters
|
| 574 |
+
INFO:hf-to-gguf:gguf: context length = 131072
|
| 575 |
+
INFO:hf-to-gguf:gguf: embedding length = 1536
|
| 576 |
+
INFO:hf-to-gguf:gguf: feed forward length = 4608
|
| 577 |
+
INFO:hf-to-gguf:gguf: head count = 16
|
| 578 |
+
INFO:hf-to-gguf:gguf: key-value head count = 1
|
| 579 |
+
INFO:hf-to-gguf:gguf: rope theta = 6000000
|
| 580 |
+
INFO:hf-to-gguf:gguf: rms norm epsilon = 1e-06
|
| 581 |
+
INFO:hf-to-gguf:gguf: expert count = 128
|
| 582 |
+
INFO:hf-to-gguf:gguf: experts used count = 8
|
| 583 |
+
INFO:hf-to-gguf:gguf: expert groups count = 8
|
| 584 |
+
INFO:hf-to-gguf:gguf: expert groups used count = 4
|
| 585 |
+
INFO:hf-to-gguf:gguf: expert score gating function = sigmoid
|
| 586 |
+
INFO:hf-to-gguf:gguf: file type = 32
|
| 587 |
+
WARNING:gguf.gguf_writer:Duplicated key name 'bailingmoe3.attention.head_count_kv', overwriting it with new value [0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1] of type ARRAY
|
| 588 |
+
WARNING:gguf.gguf_writer:Duplicated key name 'bailingmoe3.attention.key_length', overwriting it with new value 576 of type UINT32
|
| 589 |
+
INFO:hf-to-gguf:Set model quantization version
|
| 590 |
+
INFO:hf-to-gguf:Set model tokenizer
|
| 591 |
+
INFO:gguf.vocab:Adding 156635 merge(s).
|
| 592 |
+
INFO:gguf.vocab:Setting special token type bos to 156891
|
| 593 |
+
INFO:gguf.vocab:Setting special token type eos to 156895
|
| 594 |
+
INFO:gguf.vocab:Setting special token type pad to 156892
|
| 595 |
+
WARNING:gguf.vocab:No handler for special token type cls with id 156893 - skipping
|
| 596 |
+
INFO:gguf.vocab:Setting add_bos_token to False
|
| 597 |
+
INFO:gguf.vocab:Setting add_eos_token to False
|
| 598 |
+
INFO:gguf.vocab:Setting chat_template to {#- Bailing V3 chat template -#}
|
| 599 |
+
{#- Supports: thinking option, tool calling -#}
|
| 600 |
+
|
| 601 |
+
{#- ==================== thinking option normalization ==================== -#}
|
| 602 |
+
{%- if enable_thinking is defined %}
|
| 603 |
+
{%- if enable_thinking %}
|
| 604 |
+
{%- set thinking_option = 'on' %}
|
| 605 |
+
{%- else %}
|
| 606 |
+
{%- set thinking_option = 'off' %}
|
| 607 |
+
{%- endif %}
|
| 608 |
+
{%- elif thinking_option is not defined %}
|
| 609 |
+
{%- set thinking_option = 'on' %}
|
| 610 |
+
{%- endif %}
|
| 611 |
+
|
| 612 |
+
{#- ==================== preserved thinking ==================== -#}
|
| 613 |
+
{% set preserved_thinking = true %}
|
| 614 |
+
|
| 615 |
+
{#- ==================== system message ==================== -#}
|
| 616 |
+
{{- '<role>SYSTEM</role>' }}
|
| 617 |
+
{%- if tools %}
|
| 618 |
+
{%- if messages[0].role == 'system' %}
|
| 619 |
+
{{- messages[0].content + '\n' }}
|
| 620 |
+
{%- endif %}
|
| 621 |
+
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
| 622 |
+
{%- for tool in tools %}
|
| 623 |
+
{{- "\n" }}
|
| 624 |
+
{{- tool | tojson }}
|
| 625 |
+
{%- endfor %}
|
| 626 |
+
{{- "\n</tools>\n\nIf none of the functions can be used, point it out. If the given question lacks the parameters required by the function, also point it out.\nIf you need to use a function, for each function call, output the function name and arguments within the following XML format:\n<tool_call>{function-name}\n<arg_key>{arg-key-1}</arg_key>\n<arg_value>{arg-value-1}</arg_value>\n<arg_key>{arg-key-2}</arg_key>\n<arg_value>{arg-value-2}</arg_value>\n...\n</tool_call>\n" }}
|
| 627 |
+
{%- if messages[0].role == 'system' and messages[0].content is string and ('detailed thinking on' in messages[0].content or 'detailed thinking off' in messages[0].content) %}
|
| 628 |
+
{{- '<|role_end|>' }}
|
| 629 |
+
{%- else %}
|
| 630 |
+
{{- 'detailed thinking ' + thinking_option + '<|role_end|>' }}
|
| 631 |
+
{%- endif %}
|
| 632 |
+
{%- else %}
|
| 633 |
+
{%- if messages[0].role == 'system' %}
|
| 634 |
+
{%- if 'detailed thinking on' in messages[0].content or 'detailed thinking off' in messages[0].content %}
|
| 635 |
+
{{- messages[0].content + '<|role_end|>' }}
|
| 636 |
+
{%- else %}
|
| 637 |
+
{{- messages[0].content + '\n' }}
|
| 638 |
+
{{- 'detailed thinking ' + thinking_option + '<|role_end|>' }}
|
| 639 |
+
{%- endif %}
|
| 640 |
+
{% else %}
|
| 641 |
+
{{- 'detailed thinking ' + thinking_option + '<|role_end|>' }}
|
| 642 |
+
{%- endif %}
|
| 643 |
+
{%- endif %}
|
| 644 |
+
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
| 645 |
+
{%- for message in messages[::-1] %}
|
| 646 |
+
{%- set index = (messages|length - 1) - loop.index0 %}
|
| 647 |
+
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
| 648 |
+
{%- set ns.multi_step_tool = false %}
|
| 649 |
+
{%- set ns.last_query_index = index %}
|
| 650 |
+
{%- endif %}
|
| 651 |
+
{%- endfor %}
|
| 652 |
+
{%- for message in messages %}
|
| 653 |
+
{%- if message.content is string %}
|
| 654 |
+
{%- set content = message.content %}
|
| 655 |
+
{%- else %}
|
| 656 |
+
{%- set content = '' %}
|
| 657 |
+
{%- endif %}
|
| 658 |
+
{%- if message.role == "user" %}
|
| 659 |
+
{{- '<role>HUMAN</role>' + message.content + '<|role_end|>' }}
|
| 660 |
+
{%- elif message.role == "system" and not loop.first %}
|
| 661 |
+
{{- '<role>SYSTEM</role>' + message.content + '<|role_end|>' }}
|
| 662 |
+
{%- elif message.role == "assistant" %}
|
| 663 |
+
{%- set reasoning_content = '' %}
|
| 664 |
+
{%- if message.reasoning_content is string and message.reasoning_content != '' %}
|
| 665 |
+
{%- set reasoning_content = message.reasoning_content %}
|
| 666 |
+
{%- else %}
|
| 667 |
+
{%- if '</think>' in content %}
|
| 668 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 669 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 670 |
+
{%- endif %}
|
| 671 |
+
{%- endif %}
|
| 672 |
+
{%- if preserved_thinking or loop.index0 > ns.last_query_index %}
|
| 673 |
+
{%- if reasoning_content != '' %}
|
| 674 |
+
{{- '<role>ASSISTANT</role>' + '\n<think>' + reasoning_content.strip('\n') + '</think>' + content.lstrip('\n') }}
|
| 675 |
+
{%- else %}
|
| 676 |
+
{{- '<role>ASSISTANT</role>\n<think></think>' + content }}
|
| 677 |
+
{%- endif %}
|
| 678 |
+
{%- else %}
|
| 679 |
+
{{- '<role>ASSISTANT</role>\n<think></think>' + content }}
|
| 680 |
+
{%- endif %}
|
| 681 |
+
{%- if message.tool_calls %}
|
| 682 |
+
{%- for tool_call in message.tool_calls %}
|
| 683 |
+
{%- if (loop.first and content) or (not loop.first) %}
|
| 684 |
+
{{- '\n' }}
|
| 685 |
+
{%- endif %}
|
| 686 |
+
{%- set tc = tool_call %}
|
| 687 |
+
{%- if tool_call.function %}
|
| 688 |
+
{%- set tc = tool_call.function %}
|
| 689 |
+
{%- endif %}
|
| 690 |
+
{{- '<tool_call>' + tc.name }}
|
| 691 |
+
{% set _args = tc.arguments %}
|
| 692 |
+
{%- for k, v in _args.items() %}
|
| 693 |
+
{{- '<arg_key>' + k + '</arg_key>' }}
|
| 694 |
+
{{- '\n<arg_value>' }}
|
| 695 |
+
{%- if v is string %}
|
| 696 |
+
{{- v }}
|
| 697 |
+
{%- else %}
|
| 698 |
+
{{- v | tojson(ensure_ascii=False) }}
|
| 699 |
+
{%- endif %}
|
| 700 |
+
{{- '</arg_value>' }}
|
| 701 |
+
{%- endfor %}
|
| 702 |
+
{{- '\n</tool_call>' }}
|
| 703 |
+
{%- endfor %}
|
| 704 |
+
{%- endif %}
|
| 705 |
+
{{- '<|role_end|>' }}
|
| 706 |
+
{%- elif message.role == "tool" %}
|
| 707 |
+
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
| 708 |
+
{{- '<role>OBSERVATION</role>' }}
|
| 709 |
+
{%- endif %}
|
| 710 |
+
{{- '\n<tool_response>\n' }}
|
| 711 |
+
{{- content }}
|
| 712 |
+
{{- '\n</tool_response>' }}
|
| 713 |
+
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
| 714 |
+
{{- '<|role_end|>' }}
|
| 715 |
+
{%- endif %}
|
| 716 |
+
{%- endif %}
|
| 717 |
+
{%- endfor %}
|
| 718 |
+
|
| 719 |
+
{#- ==================== generation prompt ==================== -#}
|
| 720 |
+
{%- if add_generation_prompt %}
|
| 721 |
+
{{- '<role>ASSISTANT</role>' }}
|
| 722 |
+
{%- if thinking_option == 'on' %}
|
| 723 |
+
{{- '\n<think>' }}
|
| 724 |
+
{%- elif thinking_option == 'off' %}
|
| 725 |
+
{{- '\n<think></think>' }}
|
| 726 |
+
{%- endif %}
|
| 727 |
+
{%- endif %}
|
| 728 |
+
INFO:gguf.vocab:Setting normalizer_nfc to True
|
| 729 |
+
INFO:gguf.vocab:Setting clean_spaces to False
|
| 730 |
+
INFO:gguf.gguf_writer:Writing the following files:
|
| 731 |
+
INFO:gguf.gguf_writer:/tmp/Ling-3.0-tiny-hfconv.gguf: n_tensors = 526, total_size = 15.8G
|
| 732 |
+
|
| 733 |
+
INFO:hf-to-gguf:Model successfully exported to /tmp/Ling-3.0-tiny-hfconv.gguf
|
results/gen_bf16_1.txt
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#### PROMPT: 請用三句話解釋什麼是向量量化(vector quantization),並說明它與純量量化的差異。
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
Loading model... |-\|/-\|/-\|/-\|/-\|/-\|/
|
| 5 |
+
|
| 6 |
+
▄▄ ▄▄
|
| 7 |
+
██ ██
|
| 8 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 9 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 10 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 11 |
+
██ ██
|
| 12 |
+
▀▀ ▀▀
|
| 13 |
+
|
| 14 |
+
build : b1-61dbf49
|
| 15 |
+
model : models/Ling-3.0-tiny-bf16.gguf
|
| 16 |
+
ftype : BF16
|
| 17 |
+
modalities : text
|
| 18 |
+
|
| 19 |
+
available commands:
|
| 20 |
+
/exit or Ctrl+C stop or exit
|
| 21 |
+
/regen regenerate the last response
|
| 22 |
+
/clear clear the chat history
|
| 23 |
+
/read <file> add a text file
|
| 24 |
+
/glob <pattern> add text files using globbing pattern
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
> 請用三句話解釋什麼是向量量化(vector quantization),並說明它與純量量化的差異。
|
| 29 |
+
|
| 30 |
+
[Start thinking]
|
| 31 |
+
|
| 32 |
+
首先,用户要求用三句话解释向量量化(vector quantization),并说明它与纯量量化的差异。
|
| 33 |
+
|
| 34 |
+
关键点:
|
| 35 |
+
- 向量量化(vector quantization):这是一种将高维数据点映射到离散码本中的技术,常用于压缩和分类。
|
| 36 |
+
- 纯量量化(scalar quantization):这是一种将单个数值映射到离散值的技术。
|
| 37 |
+
|
| 38 |
+
我需要用三句话来解释:
|
| 39 |
+
1. 第一句:定义向量量化。
|
| 40 |
+
2. 第二句:解释向量量化的应用或特点。
|
| 41 |
+
3. 第三句:说明与纯量量化的差异。
|
| 42 |
+
|
| 43 |
+
确保语言简洁明了。
|
| 44 |
+
|
| 45 |
+
草拟三句话:
|
| 46 |
+
1. 向量量化是一种将高维数据点映射到离散码本的技术,常用于数据压缩和分类
|
| 47 |
+
|
| 48 |
+
[ Prompt: 150.0 t/s | Generation: 26.2 t/s ]
|
| 49 |
+
|
| 50 |
+
>
|
| 51 |
+
|
| 52 |
+
Exiting...
|
results/gen_bf16_2.txt
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#### PROMPT: The following is a short paragraph about the history of computing:
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
Loading model... |-\|/-\|/-\|/-\|/-\|/-
|
| 5 |
+
|
| 6 |
+
▄▄ ▄▄
|
| 7 |
+
██ ██
|
| 8 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 9 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 10 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 11 |
+
██ ██
|
| 12 |
+
▀▀ ▀▀
|
| 13 |
+
|
| 14 |
+
build : b1-61dbf49
|
| 15 |
+
model : models/Ling-3.0-tiny-bf16.gguf
|
| 16 |
+
ftype : BF16
|
| 17 |
+
modalities : text
|
| 18 |
+
|
| 19 |
+
available commands:
|
| 20 |
+
/exit or Ctrl+C stop or exit
|
| 21 |
+
/regen regenerate the last response
|
| 22 |
+
/clear clear the chat history
|
| 23 |
+
/read <file> add a text file
|
| 24 |
+
/glob <pattern> add text files using globbing pattern
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
> The following is a short paragraph about the history of computing:
|
| 29 |
+
|
| 30 |
+
[Start thinking]
|
| 31 |
+
|
| 32 |
+
The user has provided a statement that says "The following is a short paragraph about the history of computing:" but hasn't actually provided the paragraph. It seems like they're setting up a prompt but forgot to include the actual paragraph. I should let them know that the paragraph hasn't been provided yet.
|
| 33 |
+
[End thinking]
|
| 34 |
+
|
| 35 |
+
It looks like the paragraph about the history of computing wasn't included in your message. Could you please provide the short paragraph so I can help you with it?
|
| 36 |
+
|
| 37 |
+
[ Prompt: 114.9 t/s | Generation: 26.9 t/s ]
|
| 38 |
+
|
| 39 |
+
>
|
| 40 |
+
|
| 41 |
+
Exiting...
|
results/gen_bf16_3.txt
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#### PROMPT: 請將這句話翻譯成英文:「這款處理器具備 AVX-512 指令集,能加速大型語言模型的推論」。
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
Loading model... |-\|/-\|/-\|/-\|/-\|/-
|
| 5 |
+
|
| 6 |
+
▄▄ ▄▄
|
| 7 |
+
██ ██
|
| 8 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 9 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 10 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 11 |
+
██ ██
|
| 12 |
+
▀▀ ▀▀
|
| 13 |
+
|
| 14 |
+
build : b1-61dbf49
|
| 15 |
+
model : models/Ling-3.0-tiny-bf16.gguf
|
| 16 |
+
ftype : BF16
|
| 17 |
+
modalities : text
|
| 18 |
+
|
| 19 |
+
available commands:
|
| 20 |
+
/exit or Ctrl+C stop or exit
|
| 21 |
+
/regen regenerate the last response
|
| 22 |
+
/clear clear the chat history
|
| 23 |
+
/read <file> add a text file
|
| 24 |
+
/glob <pattern> add text files using globbing pattern
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
> 請將這句話翻譯成英文:「這款處理器具備 AVX-512 指令集,能加速大型語言模型的推論」。
|
| 29 |
+
|
| 30 |
+
[Start thinking]
|
| 31 |
+
|
| 32 |
+
1. **Analyze the Request:**
|
| 33 |
+
* Source text: 「這款處理器具備 AVX-512 指令集,能加速大型語言模型的推論」
|
| 34 |
+
* Target language: English
|
| 35 |
+
* Goal: Translate the sentence accurately and naturally.
|
| 36 |
+
|
| 37 |
+
2. **Deconstruct the Source Text:**
|
| 38 |
+
* 這款處理器具 (This type of processor) -> This processor / This chip / This hardware
|
| 39 |
+
* 備 (has / equipped with) -> equipped with / features
|
| 40 |
+
* AVX-512 指令集 (AVX-512 instruction set) -> AVX-512 instruction set
|
| 41 |
+
* 能加速 (can accelerate / is
|
| 42 |
+
|
| 43 |
+
[ Prompt: 162.5 t/s | Generation: 27.0 t/s ]
|
| 44 |
+
|
| 45 |
+
>
|
| 46 |
+
|
| 47 |
+
Exiting...
|
results/gen_vq3_1.txt
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#### PROMPT: 請用三句話解釋什麼是向量量化(vector quantization),並說明它與純量量化的差異。
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
Loading model... |-\|/-\|/-\|/-\|/
|
| 5 |
+
|
| 6 |
+
▄▄ ▄▄
|
| 7 |
+
██ ██
|
| 8 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 9 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 10 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 11 |
+
██ ██
|
| 12 |
+
▀▀ ▀▀
|
| 13 |
+
|
| 14 |
+
build : b1-61dbf49
|
| 15 |
+
model : models/Ling-3.0-tiny-VQ3.gguf
|
| 16 |
+
ftype : Q8_0
|
| 17 |
+
modalities : text
|
| 18 |
+
|
| 19 |
+
available commands:
|
| 20 |
+
/exit or Ctrl+C stop or exit
|
| 21 |
+
/regen regenerate the last response
|
| 22 |
+
/clear clear the chat history
|
| 23 |
+
/read <file> add a text file
|
| 24 |
+
/glob <pattern> add text files using globbing pattern
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
> 請用三句話解釋什麼是向量量化(vector quantization),並說明它與純量量化的差異。
|
| 29 |
+
|
| 30 |
+
[Start thinking]
|
| 31 |
+
|
| 32 |
+
1. **分析请求:**
|
| 33 |
+
* **主题:** 向量量化(Vector Quantization)及其与纯量量化(Scalar Quantization)的差异。
|
| 34 |
+
* **约束:** 恰好使用三句话。
|
| 35 |
+
* **语言:** 中文。
|
| 36 |
+
|
| 37 |
+
2. **解构主题:**
|
| 38 |
+
* *向量量化:* 将数据表示为向量(多维空间),并使用量化码本将这些向量映射到最接近的码本向量。它保留了多维结构。
|
| 39 |
+
* *纯量量化:* 将数据表示为标量(一维),并使用量化码本将标量映射到最接近的标量值。它只处理单一维度。
|
| 40 |
+
* *差异:* 维度(多 vs. 单),保留的结构
|
| 41 |
+
|
| 42 |
+
[ Prompt: 150.0 t/s | Generation: 32.4 t/s ]
|
| 43 |
+
|
| 44 |
+
>
|
| 45 |
+
|
| 46 |
+
Exiting...
|
results/gen_vq3_2.txt
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#### PROMPT: The following is a short paragraph about the history of computing:
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
Loading model... |-\|/-\|/-\|/-\|/
|
| 5 |
+
|
| 6 |
+
▄▄ ▄▄
|
| 7 |
+
██ ██
|
| 8 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 9 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 10 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 11 |
+
██ ██
|
| 12 |
+
▀▀ ▀▀
|
| 13 |
+
|
| 14 |
+
build : b1-61dbf49
|
| 15 |
+
model : models/Ling-3.0-tiny-VQ3.gguf
|
| 16 |
+
ftype : Q8_0
|
| 17 |
+
modalities : text
|
| 18 |
+
|
| 19 |
+
available commands:
|
| 20 |
+
/exit or Ctrl+C stop or exit
|
| 21 |
+
/regen regenerate the last response
|
| 22 |
+
/clear clear the chat history
|
| 23 |
+
/read <file> add a text file
|
| 24 |
+
/glob <pattern> add text files using globbing pattern
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
> The following is a short paragraph about the history of computing:
|
| 29 |
+
|
| 30 |
+
[Start thinking]
|
| 31 |
+
|
| 32 |
+
Hmm, the user has provided a short paragraph about the history of computing but hasn't actually included the paragraph in their message. They might have forgotten to paste it, or perhaps they're testing if I can handle a general request about computing history.
|
| 33 |
+
|
| 34 |
+
I should acknowledge the incomplete input and ask for the paragraph to be provided. Since the topic is history of computing, I can offer to discuss it once I have the content.
|
| 35 |
+
|
| 36 |
+
I'll keep my response friendly and open-ended, inviting the user to share the missing information. No need to overcomplicate it—just a simple request for the paragraph.
|
| 37 |
+
[End thinking]
|
| 38 |
+
|
| 39 |
+
It looks like you haven't included the paragraph in your message. Could you please provide the short paragraph about the history of computing? I'd be happy to discuss it with
|
| 40 |
+
|
| 41 |
+
[ Prompt: 139.8 t/s | Generation: 33.0 t/s ]
|
| 42 |
+
|
| 43 |
+
>
|
| 44 |
+
|
| 45 |
+
Exiting...
|
results/gen_vq3_3.txt
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#### PROMPT: 請將這句話翻譯成英文:「這款處理器具備 AVX-512 指令集,能加速大型語言模型的推論」。
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
Loading model... |-\|/-\|/-\|/-\|/
|
| 5 |
+
|
| 6 |
+
▄▄ ▄▄
|
| 7 |
+
██ ██
|
| 8 |
+
██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄
|
| 9 |
+
██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██
|
| 10 |
+
██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀
|
| 11 |
+
██ ██
|
| 12 |
+
▀▀ ▀▀
|
| 13 |
+
|
| 14 |
+
build : b1-61dbf49
|
| 15 |
+
model : models/Ling-3.0-tiny-VQ3.gguf
|
| 16 |
+
ftype : Q8_0
|
| 17 |
+
modalities : text
|
| 18 |
+
|
| 19 |
+
available commands:
|
| 20 |
+
/exit or Ctrl+C stop or exit
|
| 21 |
+
/regen regenerate the last response
|
| 22 |
+
/clear clear the chat history
|
| 23 |
+
/read <file> add a text file
|
| 24 |
+
/glob <pattern> add text files using globbing pattern
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
> 請將這句話翻譯成英文:「這款處理器具備 AVX-512 指令集,能加速大型語言模型的推論」。
|
| 29 |
+
|
| 30 |
+
[Start thinking]
|
| 31 |
+
|
| 32 |
+
1. **分析请求:**
|
| 33 |
+
* 目标语言:中文。
|
| 34 |
+
* 源文本:「這款處理器具備 AVX-512 指令集,能加速大型語言模型的推論」。
|
| 35 |
+
* 目标语言:英文。
|
| 36 |
+
|
| 37 |
+
2. **分析源文本:**
|
| 38 |
+
* 「這款處理器具備 AVX-512 指令集」 -> This processor has AVX-512 instructions. (或 This processor is equipped with AVX-512 instructions.)
|
| 39 |
+
* 「能加速大型語言模型的推論」 -> It can accelerate the inference of large language models. (或 It can speed up the inference of large language models.)
|
| 40 |
+
|
| 41 |
+
3.
|
| 42 |
+
|
| 43 |
+
[ Prompt: 160.9 t/s | Generation: 32.9 t/s ]
|
| 44 |
+
|
| 45 |
+
>
|
| 46 |
+
|
| 47 |
+
Exiting...
|
results/kld_base_q8dq6.log
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
0.00.362.375 W load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
|
| 5 |
+
0.01.045.232 I cmn init: llama threadpool init, n_threads = 64
|
| 6 |
+
0.01.108.713 I
|
| 7 |
+
0.01.108.894 I system_info: n_threads = 64 (n_threads_batch = 64) / 80 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
| 8 |
+
0.01.165.035 I kl_divergence: computing over 128 chunks, n_ctx=512, batch_size=2048, n_seq=4
|
| 9 |
+
0.06.918.801 I kl_divergence: 5.75 seconds per pass - ETA 3.07 minutes
|
| 10 |
+
|
| 11 |
+
chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p
|
| 12 |
+
1 8.1760 ± 1.5742 0.00150 ± 0.00906 0.00696 ± 0.00080 2.724 ± 0.353 % 95.294 ± 1.329 %
|
| 13 |
+
2 12.5990 ± 1.8201 -0.00201 ± 0.00754 0.00807 ± 0.00093 2.660 ± 0.389 % 95.686 ± 0.901 %
|
| 14 |
+
3 11.8685 ± 1.3967 0.00820 ± 0.00972 0.00958 ± 0.00095 2.886 ± 0.266 % 95.425 ± 0.756 %
|
| 15 |
+
4 12.3912 ± 1.2376 0.00823 ± 0.00798 0.00973 ± 0.00075 2.820 ± 0.216 % 95.490 ± 0.650 %
|
| 16 |
+
5 12.6999 ± 1.1133 0.00733 ± 0.00665 0.01017 ± 0.00065 2.714 ± 0.184 % 95.216 ± 0.598 %
|
| 17 |
+
6 13.1507 ± 1.0357 0.00622 ± 0.00574 0.00974 ± 0.00058 2.822 ± 0.204 % 95.033 ± 0.556 %
|
| 18 |
+
7 13.8532 ± 1.0113 0.00669 ± 0.00512 0.00953 ± 0.00051 2.777 ± 0.187 % 95.182 ± 0.507 %
|
| 19 |
+
8 14.2408 ± 0.9815 0.00497 ± 0.00494 0.01061 ± 0.00057 3.066 ± 0.219 % 95.098 ± 0.478 %
|
| 20 |
+
9 15.0203 ± 0.9801 0.00892 ± 0.00454 0.01035 ± 0.00051 2.976 ± 0.201 % 94.902 ± 0.459 %
|
| 21 |
+
10 15.5944 ± 0.9678 0.00630 ± 0.00442 0.01086 ± 0.00049 2.936 ± 0.186 % 94.784 ± 0.440 %
|
| 22 |
+
11 15.8327 ± 0.9385 0.00765 ± 0.00414 0.01067 ± 0.00046 2.895 ± 0.173 % 94.902 ± 0.415 %
|
| 23 |
+
12 16.0859 ± 0.9200 0.00680 ± 0.00409 0.01156 ± 0.00050 3.104 ± 0.206 % 94.902 ± 0.398 %
|
| 24 |
+
13 16.1902 ± 0.8854 0.00914 ± 0.00389 0.01168 ± 0.00047 3.094 ± 0.194 % 94.751 ± 0.387 %
|
| 25 |
+
14 15.8492 ± 0.8348 0.00688 ± 0.00376 0.01206 ± 0.00046 3.154 ± 0.183 % 94.678 ± 0.376 %
|
| 26 |
+
15 15.4813 ± 0.7901 0.00661 ± 0.00361 0.01189 ± 0.00044 3.118 ± 0.174 % 94.876 ± 0.357 %
|
| 27 |
+
16 15.9741 ± 0.7951 0.00834 ± 0.00361 0.01281 ± 0.00047 3.283 ± 0.170 % 94.804 ± 0.348 %
|
| 28 |
+
17 14.6659 ± 0.7006 0.00924 ± 0.00343 0.01233 ± 0.00044 3.225 ± 0.163 % 94.994 ± 0.331 %
|
| 29 |
+
18 14.7784 ± 0.6920 0.00852 ± 0.00335 0.01244 ± 0.00044 3.289 ± 0.160 % 95.054 ± 0.320 %
|
| 30 |
+
19 14.9111 ± 0.6827 0.00756 ± 0.00328 0.01253 ± 0.00042 3.286 ± 0.152 % 94.985 ± 0.314 %
|
| 31 |
+
20 14.5869 ± 0.6569 0.00833 ± 0.00331 0.01316 ± 0.00046 3.358 ± 0.145 % 95.000 ± 0.305 %
|
| 32 |
+
21 14.3900 ± 0.6299 0.00610 ± 0.00328 0.01347 ± 0.00046 3.391 ± 0.140 % 94.790 ± 0.304 %
|
| 33 |
+
22 14.1716 ± 0.6053 0.00575 ± 0.00316 0.01332 ± 0.00044 3.344 ± 0.136 % 94.742 ± 0.298 %
|
| 34 |
+
23 13.6994 ± 0.5685 0.00495 ± 0.00308 0.01323 ± 0.00043 3.378 ± 0.133 % 94.714 ± 0.292 %
|
| 35 |
+
24 13.2975 ± 0.5390 0.00515 ± 0.00306 0.01359 ± 0.00043 3.452 ± 0.132 % 94.657 ± 0.287 %
|
| 36 |
+
25 12.9230 ± 0.5133 0.00513 ± 0.00307 0.01433 ± 0.00046 3.555 ± 0.144 % 94.573 ± 0.284 %
|
| 37 |
+
26 12.7526 ± 0.4972 0.00542 ± 0.00302 0.01442 ± 0.00045 3.566 ± 0.140 % 94.585 ± 0.278 %
|
| 38 |
+
27 12.1262 ± 0.4606 0.00493 ± 0.00297 0.01439 ± 0.00045 3.601 ± 0.138 % 94.670 ± 0.271 %
|
| 39 |
+
28 11.9874 ± 0.4465 0.00546 ± 0.00296 0.01507 ± 0.00046 3.715 ± 0.138 % 94.538 ± 0.269 %
|
| 40 |
+
29 12.0150 ± 0.4408 0.00599 ± 0.00305 0.01554 ± 0.00046 3.716 ± 0.135 % 94.442 ± 0.266 %
|
| 41 |
+
30 11.7953 ± 0.4234 0.00500 ± 0.00299 0.01541 ± 0.00045 3.739 ± 0.132 % 94.471 ± 0.261 %
|
| 42 |
+
31 11.9487 ± 0.4237 0.00612 ± 0.00293 0.01544 ± 0.00044 3.748 ± 0.133 % 94.522 ± 0.256 %
|
| 43 |
+
32 11.9482 �� 0.4174 0.00508 ± 0.00288 0.01550 ± 0.00044 3.741 ± 0.130 % 94.571 ± 0.251 %
|
| 44 |
+
33 12.0238 ± 0.4146 0.00452 ± 0.00285 0.01564 ± 0.00043 3.714 ± 0.127 % 94.569 ± 0.247 %
|
| 45 |
+
34 12.2895 ± 0.4215 0.00556 ± 0.00293 0.01599 ± 0.00043 3.709 ± 0.124 % 94.521 ± 0.244 %
|
| 46 |
+
35 12.3914 ± 0.4198 0.00527 ± 0.00288 0.01605 ± 0.00042 3.732 ± 0.123 % 94.510 ± 0.241 %
|
| 47 |
+
36 12.5866 ± 0.4218 0.00583 ± 0.00288 0.01628 ± 0.00041 3.746 ± 0.121 % 94.477 ± 0.238 %
|
| 48 |
+
37 12.7177 ± 0.4197 0.00705 ± 0.00287 0.01652 ± 0.00053 3.830 ± 0.163 % 94.499 ± 0.235 %
|
| 49 |
+
38 12.8833 ± 0.4195 0.00648 ± 0.00283 0.01660 ± 0.00052 3.837 ± 0.159 % 94.448 ± 0.233 %
|
| 50 |
+
39 12.8102 ± 0.4118 0.00506 ± 0.00279 0.01657 ± 0.00051 3.846 ± 0.155 % 94.439 ± 0.230 %
|
| 51 |
+
40 12.7511 ± 0.4054 0.00515 ± 0.00274 0.01646 ± 0.00050 3.834 ± 0.152 % 94.461 ± 0.227 %
|
| 52 |
+
41 12.7324 ± 0.3996 0.00514 ± 0.00269 0.01622 ± 0.00049 3.813 ± 0.150 % 94.510 ± 0.223 %
|
| 53 |
+
42 12.6439 ± 0.3914 0.00541 ± 0.00264 0.01606 ± 0.00048 3.788 ± 0.147 % 94.463 ± 0.221 %
|
| 54 |
+
43 12.7793 ± 0.3914 0.00481 ± 0.00261 0.01592 ± 0.00047 3.770 ± 0.145 % 94.473 ± 0.218 %
|
| 55 |
+
44 12.7220 ± 0.3851 0.00465 ± 0.00256 0.01583 ± 0.00046 3.773 ± 0.142 % 94.483 ± 0.216 %
|
| 56 |
+
45 12.6415 ± 0.3775 0.00395 ± 0.00253 0.01582 ± 0.00045 3.774 ± 0.140 % 94.510 ± 0.213 %
|
| 57 |
+
46 12.8120 ± 0.3794 0.00345 ± 0.00251 0.01579 ± 0.00044 3.760 ± 0.138 % 94.518 ± 0.210 %
|
| 58 |
+
47 12.7284 ± 0.3730 0.00378 ± 0.00249 0.01595 ± 0.00046 3.836 ± 0.151 % 94.485 ± 0.209 %
|
| 59 |
+
48 12.9300 ± 0.3756 0.00367 ± 0.00246 0.01596 ± 0.00045 3.843 ± 0.149 % 94.477 ± 0.206 %
|
| 60 |
+
49 12.9314 ± 0.3710 0.00339 ± 0.00242 0.01583 ± 0.00045 3.828 ± 0.147 % 94.502 ± 0.204 %
|
| 61 |
+
50 12.9537 ± 0.3682 0.00397 ± 0.00239 0.01577 ± 0.00044 3.815 ± 0.145 % 94.494 ± 0.202 %
|
| 62 |
+
51 13.1195 ± 0.3703 0.00505 ± 0.00240 0.01599 ± 0.00044 3.807 ± 0.142 % 94.487 ± 0.200 %
|
| 63 |
+
52 13.1675 ± 0.3685 0.00509 ± 0.00238 0.01613 ± 0.00044 3.809 ± 0.140 % 94.427 ± 0.199 %
|
| 64 |
+
53 13.1573 ± 0.3646 0.00527 ± 0.00235 0.01609 ± 0.00043 3.805 ± 0.138 % 94.399 ± 0.198 %
|
| 65 |
+
54 13.1967 ± 0.3621 0.00531 ± 0.00232 0.01604 ± 0.00043 3.808 ± 0.137 % 94.408 ± 0.196 %
|
| 66 |
+
55 13.3013 ± 0.3617 0.00472 ± 0.00230 0.01599 ± 0.00042 3.810 ± 0.135 % 94.410 ± 0.194 %
|
| 67 |
+
56 13.3306 ± 0.3588 0.00496 ± 0.00227 0.01594 ± 0.00042 3.791 ± 0.134 % 94.384 ± 0.193 %
|
| 68 |
+
57 13.3631 ± 0.3563 0.00510 ± 0.00224 0.01587 ± 0.00041 3.782 ± 0.132 % 94.427 ± 0.190 %
|
| 69 |
+
58 13.3592 ± 0.3526 0.00518 ± 0.00221 0.01577 ± 0.00040 3.769 ± 0.130 % 94.429 ± 0.189 %
|
| 70 |
+
59 13.4359 ± 0.3518 0.00543 ± 0.00220 0.01586 ± 0.00040 3.771 ± 0.128 % 94.390 ± 0.188 %
|
| 71 |
+
60 13.5147 ± 0.3509 0.00553 ± 0.00217 0.01583 ± 0.00039 3.753 ± 0.127 % 94.353 ± 0.187 %
|
| 72 |
+
61 13.6081 ± 0.3506 0.00594 ± 0.00215 0.01590 ± 0.00039 3.750 ± 0.125 % 94.349 ± 0.185 %
|
| 73 |
+
62 13.6494 ± 0.3486 0.00531 ± 0.00213 0.01595 ± 0.00039 3.748 ± 0.123 % 94.326 ± 0.184 %
|
| 74 |
+
63 13.7118 ± 0.3481 0.00524 ± 0.00212 0.01600 ± 0.00038 3.740 ± 0.122 % 94.298 ± 0.183 %
|
| 75 |
+
64 13.8436 ± 0.3489 0.00512 ± 0.00209 0.01593 ± 0.00038 3.723 ± 0.120 % 94.314 ± 0.181 %
|
| 76 |
+
65 13.9977 ± 0.3507 0.00499 ± 0.00207 0.01582 ± 0.00037 3.701 ± 0.119 % 94.335 ± 0.180 %
|
| 77 |
+
66 14.1275 ± 0.3513 0.00501 ± 0.00205 0.01579 ± 0.00037 3.695 ± 0.118 % 94.349 ± 0.178 %
|
| 78 |
+
67 14.2674 ± 0.3529 0.00589 ± 0.00205 0.01588 ± 0.00037 3.685 ± 0.117 % 94.340 ± 0.177 %
|
| 79 |
+
68 14.3898 ± 0.3535 0.00542 ± 0.00204 0.01589 ± 0.00037 3.675 ± 0.115 % 94.360 ± 0.175 %
|
| 80 |
+
69 14.3072 ± 0.3484 0.00517 ± 0.00201 0.01584 ± 0.00037 3.657 ± 0.114 % 94.379 ± 0.174 %
|
| 81 |
+
70 14.3712 ± 0.3475 0.00507 ± 0.00200 0.01581 ± 0.00036 3.645 ± 0.113 % 94.381 ± 0.172 %
|
| 82 |
+
71 14.4418 ± 0.3470 0.00490 ± 0.00199 0.01593 ± 0.00036 3.655 ± 0.111 % 94.361 ± 0.171 %
|
| 83 |
+
72 14.5141 ± 0.3465 0.00484 ± 0.00198 0.01598 ± 0.00036 3.662 ± 0.110 % 94.379 ± 0.170 %
|
| 84 |
+
73 14.6097 ± 0.3470 0.00511 ± 0.00197 0.01600 ± 0.00035 3.656 ± 0.109 % 94.392 ± 0.169 %
|
| 85 |
+
74 14.5566 ± 0.3430 0.00530 ± 0.00194 0.01597 ± 0.00035 3.662 ± 0.108 % 94.399 ± 0.167 %
|
| 86 |
+
75 14.6104 ± 0.3423 0.00527 ± 0.00193 0.01599 ± 0.00035 3.667 ± 0.107 % 94.369 ± 0.167 %
|
| 87 |
+
76 14.5355 ± 0.3382 0.00504 ± 0.00191 0.01592 ± 0.00034 3.651 ± 0.106 % 94.376 ± 0.166 %
|
| 88 |
+
77 14.4600 ± 0.3341 0.00527 ± 0.00190 0.01597 ± 0.00034 3.644 ± 0.105 % 94.372 ± 0.164 %
|
| 89 |
+
78 14.4290 ± 0.3312 0.00521 ± 0.00188 0.01590 ± 0.00034 3.635 ± 0.104 % 94.399 ± 0.163 %
|
| 90 |
+
79 14.4031 ± 0.3285 0.00610 ± 0.00187 0.01597 ± 0.00034 3.640 ± 0.103 % 94.356 ± 0.163 %
|
| 91 |
+
80 14.2481 ± 0.3229 0.00626 ± 0.00185 0.01586 ± 0.00033 3.628 ± 0.102 % 94.377 ± 0.161 %
|
| 92 |
+
81 14.1859 ± 0.3190 0.00642 ± 0.00184 0.01579 ± 0.00033 3.619 ± 0.101 % 94.398 ± 0.160 %
|
| 93 |
+
82 14.1022 ± 0.3148 0.00588 ± 0.00182 0.01572 ± 0.00032 3.615 ± 0.100 % 94.424 ± 0.159 %
|
| 94 |
+
83 14.0785 ± 0.3122 0.00587 ± 0.00181 0.01570 ± 0.00032 3.604 ± 0.099 % 94.425 ± 0.158 %
|
| 95 |
+
84 14.1117 ± 0.3111 0.00609 ± 0.00179 0.01569 ± 0.00032 3.599 ± 0.098 % 94.407 ± 0.157 %
|
| 96 |
+
85 14.1530 ± 0.3100 0.00629 ± 0.00178 0.01584 ± 0.00032 3.598 ± 0.097 % 94.339 ± 0.157 %
|
| 97 |
+
86 14.1228 ± 0.3072 0.00600 ± 0.00177 0.01582 ± 0.00031 3.599 ± 0.097 % 94.314 ± 0.156 %
|
| 98 |
+
87 14.1516 ± 0.3058 0.00605 ± 0.00176 0.01576 ± 0.00031 3.584 ± 0.096 % 94.339 ± 0.155 %
|
| 99 |
+
88 14.1239 ± 0.3034 0.00588 ± 0.00174 0.01572 ± 0.00031 3.573 ± 0.095 % 94.358 ± 0.154 %
|
| 100 |
+
89 14.1581 ± 0.3027 0.00619 ± 0.00175 0.01580 ± 0.00031 3.580 ± 0.094 % 94.329 ± 0.154 %
|
| 101 |
+
90 14.1505 ± 0.3009 0.00634 ± 0.00174 0.01581 ± 0.00030 3.583 ± 0.093 % 94.327 ± 0.153 %
|
| 102 |
+
91 14.0643 ± 0.2970 0.00640 ± 0.00173 0.01578 ± 0.00030 3.578 ± 0.092 % 94.337 ± 0.152 %
|
| 103 |
+
92 14.0201 ± 0.2943 0.00675 ± 0.00172 0.01578 ± 0.00030 3.573 ± 0.091 % 94.348 ± 0.151 %
|
| 104 |
+
93 13.9880 ± 0.2915 0.00665 ± 0.00170 0.01571 ± 0.00029 3.564 ± 0.091 % 94.362 ± 0.150 %
|
| 105 |
+
94 13.9410 ± 0.2888 0.00632 ± 0.00169 0.01579 ± 0.00029 3.559 ± 0.090 % 94.355 ± 0.149 %
|
| 106 |
+
95 13.9709 ± 0.2879 0.00625 ± 0.00168 0.01570 ± 0.00029 3.547 ± 0.089 % 94.374 ± 0.148 %
|
| 107 |
+
96 13.9707 ± 0.2864 0.00635 ± 0.00167 0.01582 ± 0.00029 3.555 ± 0.088 % 94.363 ± 0.147 %
|
| 108 |
+
97 13.9618 ± 0.2843 0.00608 ± 0.00166 0.01583 ± 0.00029 3.550 ± 0.088 % 94.352 ± 0.147 %
|
| 109 |
+
98 13.9791 ± 0.2834 0.00628 ± 0.00166 0.01585 ± 0.00029 3.554 ± 0.087 % 94.378 ± 0.146 %
|
| 110 |
+
99 13.9227 ± 0.2806 0.00615 ± 0.00165 0.01583 ± 0.00029 3.546 ± 0.087 % 94.379 ± 0.145 %
|
| 111 |
+
100 13.9290 ± 0.2792 0.00615 ± 0.00164 0.01591 ± 0.00029 3.547 ± 0.086 % 94.369 ± 0.144 %
|
| 112 |
+
101 13.8896 ± 0.2769 0.00574 ± 0.00163 0.01601 ± 0.00028 3.557 ± 0.085 % 94.351 ± 0.144 %
|
| 113 |
+
102 13.9313 ± 0.2765 0.00587 ± 0.00162 0.01598 ± 0.00028 3.551 ± 0.085 % 94.371 ± 0.143 %
|
| 114 |
+
103 13.9681 ± 0.2759 0.00549 ± 0.00161 0.01594 ± 0.00028 3.540 ± 0.084 % 94.365 ± 0.142 %
|
| 115 |
+
104 13.9554 ± 0.2743 0.00553 ± 0.00160 0.01593 ± 0.00028 3.540 ± 0.084 % 94.370 ± 0.142 %
|
| 116 |
+
105 14.0603 ± 0.2760 0.00571 ± 0.00160 0.01605 ± 0.00028 3.552 ± 0.083 % 94.360 ± 0.141 %
|
| 117 |
+
106 14.0206 ± 0.2739 0.00570 ± 0.00159 0.01608 ± 0.00028 3.555 ± 0.082 % 94.354 ± 0.140 %
|
| 118 |
+
107 14.0305 ± 0.2732 0.00614 ± 0.00159 0.01610 ± 0.00027 3.561 ± 0.082 % 94.378 ± 0.139 %
|
| 119 |
+
108 13.9946 ± 0.2710 0.00633 ± 0.00158 0.01601 ± 0.00027 3.553 ± 0.081 % 94.379 ± 0.139 %
|
| 120 |
+
109 14.0181 ± 0.2700 0.00638 ± 0.00156 0.01593 ± 0.00027 3.544 ± 0.081 % 94.402 ± 0.138 %
|
| 121 |
+
110 14.0882 ± 0.2700 0.00638 ± 0.00155 0.01583 ± 0.00027 3.531 ± 0.080 % 94.396 ± 0.137 %
|
| 122 |
+
111 14.1523 ± 0.2701 0.00622 ± 0.00154 0.01582 ± 0.00027 3.536 ± 0.080 % 94.390 ± 0.137 %
|
| 123 |
+
112 14.2709 ± 0.2714 0.00656 ± 0.00153 0.01580 ± 0.00026 3.529 ± 0.079 % 94.387 ± 0.136 %
|
| 124 |
+
113 14.4146 ± 0.2736 0.00658 ± 0.00152 0.01574 ± 0.00026 3.519 ± 0.079 % 94.388 ± 0.136 %
|
| 125 |
+
114 14.4226 ± 0.2724 0.00658 ± 0.00151 0.01568 ± 0.00026 3.510 ± 0.078 % 94.407 ± 0.135 %
|
| 126 |
+
115 14.3334 ± 0.2692 0.00671 ± 0.00150 0.01568 ± 0.00026 3.529 ± 0.081 % 94.431 ± 0.134 %
|
| 127 |
+
116 14.3994 ± 0.2693 0.00670 ± 0.00149 0.01559 ± 0.00026 3.517 ± 0.080 % 94.432 ± 0.133 %
|
| 128 |
+
117 14.4436 ± 0.2690 0.00662 ± 0.00149 0.01557 ± 0.00026 3.514 ± 0.080 % 94.423 ± 0.133 %
|
| 129 |
+
118 14.4301 ± 0.2676 0.00647 ± 0.00148 0.01555 ± 0.00026 3.513 ± 0.080 % 94.427 ± 0.132 %
|
| 130 |
+
119 14.3979 ± 0.2659 0.00675 ± 0.00147 0.01557 ± 0.00026 3.514 ± 0.079 % 94.424 ± 0.132 %
|
| 131 |
+
120 14.3522 ± 0.2639 0.00677 ± 0.00146 0.01550 ± 0.00025 3.506 ± 0.078 % 94.431 ± 0.131 %
|
| 132 |
+
121 14.3122 ± 0.2619 0.00674 ± 0.00145 0.01544 ± 0.00025 3.498 ± 0.078 % 94.448 ± 0.130 %
|
| 133 |
+
122 14.2960 ± 0.2604 0.00662 ± 0.00144 0.01537 ± 0.00025 3.489 ± 0.078 % 94.468 ± 0.130 %
|
| 134 |
+
123 14.2923 ± 0.2591 0.00656 ± 0.00143 0.01531 ± 0.00025 3.480 ± 0.077 % 94.478 ± 0.129 %
|
| 135 |
+
124 14.2747 ± 0.2577 0.00641 ± 0.00142 0.01526 ± 0.00025 3.481 ± 0.077 % 94.497 ± 0.128 %
|
| 136 |
+
125 14.2259 ± 0.2559 0.00616 ± 0.00142 0.01527 ± 0.00024 3.484 ± 0.077 % 94.497 ± 0.128 %
|
| 137 |
+
126 14.1607 ± 0.2537 0.00598 ± 0.00141 0.01529 ± 0.00024 3.484 ± 0.076 % 94.507 ± 0.127 %
|
| 138 |
+
127 14.1116 ± 0.2517 0.00578 ± 0.00140 0.01525 ± 0.00024 3.475 ± 0.076 % 94.510 ± 0.127 %
|
| 139 |
+
128 14.0805 ± 0.2499 0.00559 ± 0.00140 0.01519 ± 0.00024 3.467 ± 0.075 % 94.531 ± 0.126 %
|
| 140 |
+
|
| 141 |
+
====== Perplexity statistics ======
|
| 142 |
+
Mean PPL(Q) : 14.080488 ± 0.249930
|
| 143 |
+
Mean PPL(base) : 14.001939 ± 0.247589
|
| 144 |
+
Cor(ln(PPL(Q)), ln(PPL(base))): 99.69%
|
| 145 |
+
Mean ln(PPL(Q)/PPL(base)) : 0.005594 ± 0.001396
|
| 146 |
+
Mean PPL(Q)/PPL(base) : 1.005610 ± 0.001404
|
| 147 |
+
Mean PPL(Q)-PPL(base) : 0.078549 ± 0.019723
|
| 148 |
+
|
| 149 |
+
====== KL divergence statistics ======
|
| 150 |
+
Mean KLD: 0.015190 ± 0.000241
|
| 151 |
+
Maximum KLD: 3.273471
|
| 152 |
+
99.9% KLD: 0.478895
|
| 153 |
+
99.0% KLD: 0.153946
|
| 154 |
+
95.0% KLD: 0.058134
|
| 155 |
+
90.0% KLD: 0.034931
|
| 156 |
+
Median KLD: 0.005134
|
| 157 |
+
10.0% KLD: 0.000063
|
| 158 |
+
5.0% KLD: 0.000009
|
| 159 |
+
1.0% KLD: -0.000000
|
| 160 |
+
0.1% KLD: -0.000011
|
| 161 |
+
Minimum KLD: -0.000220
|
| 162 |
+
|
| 163 |
+
====== Token probability statistics ======
|
| 164 |
+
Mean Δp: -0.029 ± 0.019 %
|
| 165 |
+
Maximum Δp: 57.914%
|
| 166 |
+
99.9% Δp: 24.608%
|
| 167 |
+
99.0% Δp: 10.660%
|
| 168 |
+
95.0% Δp: 4.025%
|
| 169 |
+
90.0% Δp: 2.050%
|
| 170 |
+
75.0% Δp: 0.269%
|
| 171 |
+
Median Δp: -0.000%
|
| 172 |
+
25.0% Δp: -0.298%
|
| 173 |
+
10.0% Δp: -2.083%
|
| 174 |
+
5.0% Δp: -4.143%
|
| 175 |
+
1.0% Δp: -10.916%
|
| 176 |
+
0.1% Δp: -27.079%
|
| 177 |
+
Minimum Δp: -91.157%
|
| 178 |
+
RMS Δp : 3.467 ± 0.075 %
|
| 179 |
+
Same top p: 94.531 ± 0.126 %
|
| 180 |
+
|
results/kld_q4_k_m.log
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
0.00.330.931 W load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
|
| 5 |
+
0.04.875.881 I cmn init: llama threadpool init, n_threads = 64
|
| 6 |
+
0.05.018.549 I
|
| 7 |
+
0.05.018.707 I system_info: n_threads = 64 (n_threads_batch = 64) / 80 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
| 8 |
+
0.05.069.457 I kl_divergence: computing over 128 chunks, n_ctx=512, batch_size=2048, n_seq=4
|
| 9 |
+
0.11.082.265 I kl_divergence: 6.01 seconds per pass - ETA 3.20 minutes
|
| 10 |
+
|
| 11 |
+
chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p
|
| 12 |
+
1 9.2135 ± 1.8084 0.12097 ± 0.03362 0.08542 ± 0.01081 9.386 ± 1.069 % 87.451 ± 2.079 %
|
| 13 |
+
2 13.7032 ± 2.0098 0.08200 ± 0.02443 0.07899 ± 0.00734 7.782 ± 0.697 % 87.647 ± 1.458 %
|
| 14 |
+
3 12.8484 ± 1.5211 0.08753 ± 0.02217 0.10252 ± 0.00848 9.276 ± 0.739 % 86.928 ± 1.220 %
|
| 15 |
+
4 13.3780 ± 1.3520 0.08486 ± 0.01883 0.10360 ± 0.00691 8.768 ± 0.602 % 86.078 ± 1.084 %
|
| 16 |
+
5 13.5508 ± 1.2008 0.07218 ± 0.01633 0.10521 ± 0.00577 8.442 ± 0.512 % 85.098 ± 0.998 %
|
| 17 |
+
6 13.7879 ± 1.0958 0.05354 ± 0.01427 0.09711 ± 0.00489 8.076 ± 0.458 % 85.817 ± 0.892 %
|
| 18 |
+
7 14.6299 ± 1.0809 0.06124 ± 0.01319 0.09570 ± 0.00444 7.973 ± 0.433 % 85.714 ± 0.828 %
|
| 19 |
+
8 14.9460 ± 1.0396 0.05330 ± 0.01266 0.09982 ± 0.00440 8.422 ± 0.455 % 85.147 ± 0.788 %
|
| 20 |
+
9 15.4850 ± 1.0156 0.03939 ± 0.01170 0.09550 ± 0.00397 8.135 ± 0.422 % 85.490 ± 0.735 %
|
| 21 |
+
10 16.0452 ± 1.0012 0.03480 ± 0.01158 0.09956 ± 0.00389 8.341 ± 0.424 % 84.706 ± 0.713 %
|
| 22 |
+
11 16.2375 ± 0.9667 0.03290 ± 0.01082 0.09764 ± 0.00358 8.254 ± 0.395 % 84.848 ± 0.677 %
|
| 23 |
+
12 16.4255 ± 0.9413 0.02769 ± 0.01083 0.10483 ± 0.00376 8.539 ± 0.380 % 84.673 ± 0.651 %
|
| 24 |
+
13 16.5658 ± 0.9076 0.03208 ± 0.01028 0.10273 ± 0.00352 8.489 ± 0.360 % 84.857 ± 0.623 %
|
| 25 |
+
14 16.1432 ± 0.8504 0.02526 ± 0.01013 0.10813 ± 0.00361 8.734 ± 0.342 % 84.734 ± 0.602 %
|
| 26 |
+
15 15.7884 ± 0.8055 0.02625 ± 0.00980 0.10977 ± 0.00353 9.030 ± 0.343 % 84.810 ± 0.580 %
|
| 27 |
+
16 16.2771 ± 0.8097 0.02713 ± 0.00965 0.11368 ± 0.00344 9.106 ± 0.325 % 84.436 ± 0.568 %
|
| 28 |
+
17 14.8911 ± 0.7103 0.02448 ± 0.00921 0.10997 ± 0.00326 9.028 ± 0.312 % 84.844 ± 0.545 %
|
| 29 |
+
18 14.9903 ± 0.7011 0.02276 ± 0.00888 0.11052 ± 0.00320 9.075 ± 0.303 % 85.011 ± 0.527 %
|
| 30 |
+
19 15.1307 ± 0.6910 0.02218 ± 0.00873 0.11114 ± 0.00309 9.041 ± 0.291 % 84.706 ± 0.517 %
|
| 31 |
+
20 14.7837 ± 0.6625 0.02174 ± 0.00885 0.11430 ± 0.00308 9.204 ± 0.283 % 84.784 ± 0.503 %
|
| 32 |
+
21 14.5570 ± 0.6334 0.01764 ± 0.00875 0.11597 ± 0.00308 9.241 ± 0.275 % 84.818 ± 0.490 %
|
| 33 |
+
22 14.3470 ± 0.6091 0.01805 ± 0.00853 0.11515 ± 0.00298 9.139 ± 0.266 % 84.688 ± 0.481 %
|
| 34 |
+
23 13.9338 ± 0.5761 0.02192 ± 0.00834 0.11489 ± 0.00293 9.244 ± 0.268 % 84.672 ± 0.470 %
|
| 35 |
+
24 13.5553 ± 0.5476 0.02436 ± 0.00838 0.11924 ± 0.00309 9.511 ± 0.266 % 84.444 ± 0.463 %
|
| 36 |
+
25 13.2813 ± 0.5281 0.03248 ± 0.00867 0.12441 ± 0.00323 9.885 ± 0.278 % 84.361 ± 0.455 %
|
| 37 |
+
26 13.0897 ± 0.5108 0.03151 ± 0.00855 0.12615 ± 0.00330 9.991 ± 0.275 % 84.314 ± 0.447 %
|
| 38 |
+
27 12.4494 ± 0.4736 0.03124 ± 0.00839 0.12523 ± 0.00321 10.018 ± 0.267 % 84.503 ± 0.436 %
|
| 39 |
+
28 12.3204 ± 0.4600 0.03287 ± 0.00826 0.12707 ± 0.00317 10.152 ± 0.265 % 84.482 ± 0.429 %
|
| 40 |
+
29 12.3959 ± 0.4566 0.03721 ± 0.00827 0.13083 ± 0.00319 10.190 ± 0.258 % 84.408 ± 0.422 %
|
| 41 |
+
30 12.2044 ± 0.4404 0.03910 ± 0.00812 0.13027 ± 0.00312 10.203 ± 0.253 % 84.523 ± 0.414 %
|
| 42 |
+
31 12.3413 ± 0.4393 0.03846 ± 0.00801 0.13014 ± 0.00304 10.155 ± 0.248 % 84.630 ± 0.406 %
|
| 43 |
+
32 12.3383 �� 0.4325 0.03720 ± 0.00785 0.13031 ± 0.00298 10.109 ± 0.242 % 84.645 ± 0.399 %
|
| 44 |
+
33 12.4370 ± 0.4307 0.03830 ± 0.00779 0.13120 ± 0.00293 10.106 ± 0.239 % 84.540 ± 0.394 %
|
| 45 |
+
34 12.6489 ± 0.4345 0.03438 ± 0.00777 0.13349 ± 0.00292 10.156 ± 0.234 % 84.452 ± 0.389 %
|
| 46 |
+
35 12.7494 ± 0.4326 0.03374 ± 0.00770 0.13600 ± 0.00293 10.237 ± 0.234 % 84.280 ± 0.385 %
|
| 47 |
+
36 12.9727 ± 0.4356 0.03605 ± 0.00764 0.13847 ± 0.00290 10.243 ± 0.229 % 84.172 ± 0.381 %
|
| 48 |
+
37 13.0931 ± 0.4327 0.03614 ± 0.00755 0.13810 ± 0.00291 10.205 ± 0.228 % 84.102 ± 0.376 %
|
| 49 |
+
38 13.3507 ± 0.4361 0.04212 ± 0.00757 0.14172 ± 0.00291 10.305 ± 0.227 % 83.922 ± 0.373 %
|
| 50 |
+
39 13.2475 ± 0.4267 0.03863 ± 0.00746 0.14133 ± 0.00286 10.360 ± 0.224 % 83.922 ± 0.368 %
|
| 51 |
+
40 13.1842 ± 0.4201 0.03855 ± 0.00735 0.14076 ± 0.00280 10.312 ± 0.221 % 83.931 ± 0.364 %
|
| 52 |
+
41 13.1719 ± 0.4143 0.03908 ± 0.00721 0.13919 ± 0.00274 10.238 ± 0.218 % 84.027 ± 0.358 %
|
| 53 |
+
42 13.0768 ± 0.4056 0.03908 ± 0.00709 0.13789 ± 0.00269 10.211 ± 0.214 % 84.080 ± 0.354 %
|
| 54 |
+
43 13.2066 ± 0.4053 0.03770 ± 0.00699 0.13727 ± 0.00263 10.157 ± 0.211 % 84.122 ± 0.349 %
|
| 55 |
+
44 13.1272 ± 0.3977 0.03601 ± 0.00688 0.13634 ± 0.00260 10.147 ± 0.208 % 84.162 ± 0.345 %
|
| 56 |
+
45 13.0755 ± 0.3912 0.03770 ± 0.00685 0.13767 ± 0.00258 10.250 ± 0.207 % 84.061 ± 0.342 %
|
| 57 |
+
46 13.2625 ± 0.3935 0.03801 ± 0.00677 0.13806 ± 0.00255 10.271 ± 0.205 % 84.049 ± 0.338 %
|
| 58 |
+
47 13.1535 ± 0.3862 0.03663 ± 0.00672 0.13822 ± 0.00252 10.360 ± 0.203 % 84.022 ± 0.335 %
|
| 59 |
+
48 13.3759 ± 0.3896 0.03757 ± 0.00664 0.13793 ± 0.00248 10.311 ± 0.200 % 83.979 ± 0.332 %
|
| 60 |
+
49 13.3656 ± 0.3845 0.03642 ± 0.00652 0.13625 ± 0.00243 10.255 ± 0.197 % 84.106 ± 0.327 %
|
| 61 |
+
50 13.3755 ± 0.3811 0.03601 ± 0.00646 0.13634 ± 0.00240 10.277 ± 0.195 % 84.086 ± 0.324 %
|
| 62 |
+
51 13.5047 ± 0.3817 0.03399 ± 0.00641 0.13680 ± 0.00236 10.273 ± 0.192 % 84.052 ± 0.321 %
|
| 63 |
+
52 13.5339 ± 0.3789 0.03254 ± 0.00638 0.13753 ± 0.00234 10.261 ± 0.190 % 84.035 ± 0.318 %
|
| 64 |
+
53 13.5469 ± 0.3758 0.03445 ± 0.00631 0.13752 ± 0.00231 10.292 ± 0.188 % 83.981 ± 0.316 %
|
| 65 |
+
54 13.5795 ± 0.3730 0.03391 ± 0.00622 0.13664 ± 0.00228 10.258 ± 0.185 % 83.965 ± 0.313 %
|
| 66 |
+
55 13.7141 ± 0.3736 0.03529 ± 0.00614 0.13644 ± 0.00227 10.263 ± 0.186 % 84.021 ± 0.309 %
|
| 67 |
+
56 13.7510 ± 0.3707 0.03600 ± 0.00607 0.13588 ± 0.00223 10.223 ± 0.183 % 83.971 ± 0.307 %
|
| 68 |
+
57 13.7730 ± 0.3677 0.03532 ± 0.00599 0.13522 ± 0.00222 10.162 ± 0.181 % 84.073 ± 0.304 %
|
| 69 |
+
58 13.7705 ± 0.3641 0.03551 ± 0.00594 0.13494 ± 0.00219 10.191 ± 0.179 % 84.064 ± 0.301 %
|
| 70 |
+
59 13.8400 ± 0.3629 0.03506 ± 0.00589 0.13547 ± 0.00218 10.177 ± 0.177 % 84.015 ± 0.299 %
|
| 71 |
+
60 13.9250 ± 0.3621 0.03543 ± 0.00583 0.13507 ± 0.00215 10.136 ± 0.175 % 83.974 ± 0.297 %
|
| 72 |
+
61 14.0361 ± 0.3624 0.03690 ± 0.00579 0.13603 ± 0.00214 10.137 ± 0.174 % 83.915 ± 0.295 %
|
| 73 |
+
62 14.0615 ± 0.3599 0.03505 ± 0.00573 0.13576 ± 0.00211 10.101 ± 0.172 % 83.953 ± 0.292 %
|
| 74 |
+
63 14.1002 ± 0.3584 0.03318 ± 0.00569 0.13660 ± 0.00210 10.144 ± 0.172 % 83.940 ± 0.290 %
|
| 75 |
+
64 14.2422 ± 0.3595 0.03351 ± 0.00564 0.13638 ± 0.00208 10.106 ± 0.170 % 83.873 ± 0.288 %
|
| 76 |
+
65 14.3915 ± 0.3611 0.03273 ± 0.00558 0.13558 ± 0.00205 10.056 ± 0.169 % 83.813 ± 0.286 %
|
| 77 |
+
66 14.5562 ± 0.3627 0.03490 ± 0.00552 0.13555 ± 0.00202 10.024 ± 0.167 % 83.832 ± 0.284 %
|
| 78 |
+
67 14.7186 ± 0.3650 0.03702 ± 0.00551 0.13627 ± 0.00201 10.042 ± 0.165 % 83.752 ± 0.282 %
|
| 79 |
+
68 14.8362 ± 0.3654 0.03598 ± 0.00547 0.13622 ± 0.00199 10.011 ± 0.164 % 83.812 ± 0.280 %
|
| 80 |
+
69 14.7468 ± 0.3600 0.03544 ± 0.00541 0.13512 ± 0.00196 9.962 ± 0.162 % 83.899 ± 0.277 %
|
| 81 |
+
70 14.7955 ± 0.3586 0.03417 ± 0.00536 0.13553 ± 0.00196 9.949 ± 0.161 % 83.798 ± 0.276 %
|
| 82 |
+
71 14.9094 ± 0.3592 0.03677 ± 0.00535 0.13709 ± 0.00197 10.017 ± 0.161 % 83.679 ± 0.275 %
|
| 83 |
+
72 14.9684 ± 0.3583 0.03566 ± 0.00532 0.13740 ± 0.00200 9.993 ± 0.159 % 83.676 ± 0.273 %
|
| 84 |
+
73 15.0539 ± 0.3583 0.03506 ± 0.00528 0.13766 ± 0.00199 9.973 ± 0.158 % 83.669 ± 0.271 %
|
| 85 |
+
74 15.0111 ± 0.3545 0.03604 ± 0.00525 0.13790 ± 0.00197 9.982 ± 0.157 % 83.625 ± 0.269 %
|
| 86 |
+
75 15.0635 ± 0.3538 0.03581 ± 0.00522 0.13811 ± 0.00196 10.005 ± 0.156 % 83.613 ± 0.268 %
|
| 87 |
+
76 15.0026 ± 0.3500 0.03666 ± 0.00517 0.13737 ± 0.00194 9.972 ± 0.155 % 83.643 ± 0.266 %
|
| 88 |
+
77 14.9241 ± 0.3459 0.03686 ± 0.00512 0.13708 ± 0.00192 9.963 ± 0.154 % 83.667 ± 0.264 %
|
| 89 |
+
78 14.8713 ± 0.3423 0.03540 ± 0.00508 0.13657 ± 0.00190 9.935 ± 0.153 % 83.675 ± 0.262 %
|
| 90 |
+
79 14.8516 ± 0.3396 0.03676 ± 0.00504 0.13664 ± 0.00189 9.952 ± 0.152 % 83.678 ± 0.260 %
|
| 91 |
+
80 14.6729 ± 0.3333 0.03564 ± 0.00502 0.13629 ± 0.00188 9.959 ± 0.151 % 83.750 ± 0.258 %
|
| 92 |
+
81 14.5973 ± 0.3290 0.03501 ± 0.00498 0.13586 ± 0.00187 9.940 ± 0.150 % 83.839 ± 0.256 %
|
| 93 |
+
82 14.4948 ± 0.3241 0.03334 ± 0.00495 0.13562 ± 0.00185 9.937 ± 0.149 % 83.879 ± 0.254 %
|
| 94 |
+
83 14.4599 ± 0.3211 0.03260 ± 0.00492 0.13552 ± 0.00184 9.940 ± 0.148 % 83.907 ± 0.253 %
|
| 95 |
+
84 14.4753 ± 0.3194 0.03154 ± 0.00489 0.13599 ± 0.00183 9.962 ± 0.148 % 83.875 ± 0.251 %
|
| 96 |
+
85 14.5235 ± 0.3185 0.03214 ± 0.00486 0.13676 ± 0.00182 9.991 ± 0.148 % 83.811 ± 0.250 %
|
| 97 |
+
86 14.5066 ± 0.3161 0.03282 ± 0.00484 0.13704 ± 0.00181 10.005 ± 0.147 % 83.780 ± 0.249 %
|
| 98 |
+
87 14.5267 ± 0.3145 0.03222 ± 0.00481 0.13679 ± 0.00179 9.988 ± 0.145 % 83.795 ± 0.247 %
|
| 99 |
+
88 14.5015 ± 0.3119 0.03226 ± 0.00478 0.13695 ± 0.00179 10.016 ± 0.145 % 83.815 ± 0.246 %
|
| 100 |
+
89 14.5328 ± 0.3112 0.03232 ± 0.00477 0.13782 ± 0.00178 10.022 ± 0.144 % 83.811 ± 0.245 %
|
| 101 |
+
90 14.5096 ± 0.3088 0.03140 ± 0.00474 0.13807 ± 0.00177 10.020 ± 0.143 % 83.843 ± 0.243 %
|
| 102 |
+
91 14.4196 ± 0.3049 0.03134 ± 0.00471 0.13780 ± 0.00176 10.022 ± 0.142 % 83.870 ± 0.241 %
|
| 103 |
+
92 14.3630 ± 0.3017 0.03091 ± 0.00468 0.13773 ± 0.00174 10.033 ± 0.141 % 83.875 ± 0.240 %
|
| 104 |
+
93 14.3360 ± 0.2990 0.03123 ± 0.00464 0.13695 ± 0.00173 10.004 ± 0.140 % 83.892 ± 0.239 %
|
| 105 |
+
94 14.2916 ± 0.2962 0.03116 ± 0.00461 0.13710 ± 0.00171 10.004 ± 0.139 % 83.892 ± 0.237 %
|
| 106 |
+
95 14.3150 ± 0.2951 0.03058 ± 0.00457 0.13648 ± 0.00170 9.987 ± 0.139 % 83.926 ± 0.236 %
|
| 107 |
+
96 14.3164 ± 0.2935 0.03080 ± 0.00456 0.13696 ± 0.00170 10.026 ± 0.139 % 83.954 ± 0.235 %
|
| 108 |
+
97 14.3207 ± 0.2918 0.03146 ± 0.00453 0.13664 ± 0.00168 9.996 ± 0.138 % 83.990 ± 0.233 %
|
| 109 |
+
98 14.3503 ± 0.2912 0.03248 ± 0.00450 0.13688 ± 0.00167 9.995 ± 0.137 % 84.002 ± 0.232 %
|
| 110 |
+
99 14.2947 ± 0.2883 0.03252 ± 0.00448 0.13691 ± 0.00167 9.996 ± 0.136 % 84.001 ± 0.231 %
|
| 111 |
+
100 14.2994 ± 0.2869 0.03240 ± 0.00447 0.13740 ± 0.00168 10.020 ± 0.137 % 83.973 ± 0.230 %
|
| 112 |
+
101 14.2694 ± 0.2848 0.03271 ± 0.00444 0.13742 ± 0.00167 10.003 ± 0.136 % 83.980 ± 0.229 %
|
| 113 |
+
102 14.2993 ± 0.2840 0.03195 ± 0.00441 0.13685 ± 0.00165 9.975 ± 0.135 % 84.014 ± 0.227 %
|
| 114 |
+
103 14.3255 ± 0.2832 0.03076 ± 0.00437 0.13614 ± 0.00164 9.939 ± 0.134 % 84.055 ± 0.226 %
|
| 115 |
+
104 14.3197 ± 0.2817 0.03130 ± 0.00435 0.13675 ± 0.00166 9.958 ± 0.134 % 84.031 ± 0.225 %
|
| 116 |
+
105 14.4117 ± 0.2830 0.03040 ± 0.00436 0.13769 ± 0.00166 9.967 ± 0.133 % 84.007 ± 0.224 %
|
| 117 |
+
106 14.3633 ± 0.2806 0.02985 ± 0.00433 0.13781 ± 0.00165 9.955 ± 0.133 % 84.036 ± 0.223 %
|
| 118 |
+
107 14.3511 ± 0.2792 0.02874 ± 0.00431 0.13794 ± 0.00164 9.964 ± 0.132 % 84.061 ± 0.222 %
|
| 119 |
+
108 14.3071 ± 0.2768 0.02842 ± 0.00428 0.13740 ± 0.00163 9.944 ± 0.131 % 84.096 ± 0.220 %
|
| 120 |
+
109 14.3298 ± 0.2758 0.02837 ± 0.00425 0.13687 ± 0.00162 9.924 ± 0.131 % 84.119 ± 0.219 %
|
| 121 |
+
110 14.4074 ± 0.2759 0.02878 ± 0.00423 0.13632 ± 0.00161 9.898 ± 0.130 % 84.125 ± 0.218 %
|
| 122 |
+
111 14.4750 ± 0.2761 0.02876 ± 0.00420 0.13620 ± 0.00160 9.890 ± 0.129 % 84.134 ± 0.217 %
|
| 123 |
+
112 14.5840 ± 0.2772 0.02826 ± 0.00417 0.13590 ± 0.00159 9.867 ± 0.129 % 84.128 ± 0.216 %
|
| 124 |
+
113 14.7311 ± 0.2794 0.02830 ± 0.00415 0.13555 ± 0.00158 9.842 ± 0.128 % 84.144 ± 0.215 %
|
| 125 |
+
114 14.7301 ± 0.2779 0.02768 ± 0.00412 0.13549 ± 0.00158 9.830 ± 0.127 % 84.128 ± 0.214 %
|
| 126 |
+
115 14.6281 ± 0.2745 0.02706 ± 0.00410 0.13529 ± 0.00157 9.831 ± 0.127 % 84.181 ± 0.213 %
|
| 127 |
+
116 14.6985 ± 0.2747 0.02726 ± 0.00407 0.13459 ± 0.00156 9.804 ± 0.126 % 84.222 ± 0.212 %
|
| 128 |
+
117 14.7350 ± 0.2742 0.02660 ± 0.00407 0.13473 ± 0.00156 9.829 ± 0.126 % 84.186 ± 0.211 %
|
| 129 |
+
118 14.7223 ± 0.2728 0.02652 ± 0.00405 0.13452 ± 0.00155 9.826 ± 0.126 % 84.177 ± 0.210 %
|
| 130 |
+
119 14.6785 ± 0.2708 0.02605 ± 0.00403 0.13446 ± 0.00154 9.814 ± 0.125 % 84.175 ± 0.210 %
|
| 131 |
+
120 14.6353 ± 0.2688 0.02630 ± 0.00400 0.13411 ± 0.00153 9.809 ± 0.125 % 84.193 ± 0.209 %
|
| 132 |
+
121 14.5970 ± 0.2669 0.02644 ± 0.00399 0.13421 ± 0.00153 9.798 ± 0.124 % 84.191 ± 0.208 %
|
| 133 |
+
122 14.5769 ± 0.2653 0.02608 ± 0.00397 0.13396 ± 0.00152 9.790 ± 0.124 % 84.221 ± 0.207 %
|
| 134 |
+
123 14.5777 ± 0.2641 0.02634 ± 0.00395 0.13381 ± 0.00151 9.779 ± 0.123 % 84.224 ± 0.206 %
|
| 135 |
+
124 14.5521 ± 0.2626 0.02566 ± 0.00393 0.13345 ± 0.00150 9.761 ± 0.123 % 84.257 ± 0.205 %
|
| 136 |
+
125 14.5122 ± 0.2608 0.02609 ± 0.00391 0.13342 ± 0.00149 9.755 ± 0.122 % 84.264 ± 0.204 %
|
| 137 |
+
126 14.4496 ± 0.2588 0.02617 ± 0.00390 0.13363 ± 0.00149 9.779 ± 0.122 % 84.251 ± 0.203 %
|
| 138 |
+
127 14.3935 ± 0.2566 0.02556 ± 0.00388 0.13351 ± 0.00148 9.773 ± 0.121 % 84.240 ± 0.202 %
|
| 139 |
+
128 14.3726 ± 0.2551 0.02613 ± 0.00386 0.13312 ± 0.00147 9.753 ± 0.121 % 84.292 ± 0.201 %
|
| 140 |
+
|
| 141 |
+
====== Perplexity statistics ======
|
| 142 |
+
Mean PPL(Q) : 14.372588 ± 0.255101
|
| 143 |
+
Mean PPL(base) : 14.001939 ± 0.247589
|
| 144 |
+
Cor(ln(PPL(Q)), ln(PPL(base))): 97.63%
|
| 145 |
+
Mean ln(PPL(Q)/PPL(base)) : 0.026127 ± 0.003861
|
| 146 |
+
Mean PPL(Q)/PPL(base) : 1.026471 ± 0.003963
|
| 147 |
+
Mean PPL(Q)-PPL(base) : 0.370649 ± 0.055277
|
| 148 |
+
|
| 149 |
+
====== KL divergence statistics ======
|
| 150 |
+
Mean KLD: 0.133121 ± 0.001471
|
| 151 |
+
Maximum KLD: 7.737188
|
| 152 |
+
99.9% KLD: 2.917165
|
| 153 |
+
99.0% KLD: 1.247232
|
| 154 |
+
95.0% KLD: 0.505470
|
| 155 |
+
90.0% KLD: 0.319245
|
| 156 |
+
Median KLD: 0.055794
|
| 157 |
+
10.0% KLD: 0.000690
|
| 158 |
+
5.0% KLD: 0.000112
|
| 159 |
+
1.0% KLD: 0.000006
|
| 160 |
+
0.1% KLD: -0.000002
|
| 161 |
+
Minimum KLD: -0.000250
|
| 162 |
+
|
| 163 |
+
====== Token probability statistics ======
|
| 164 |
+
Mean Δp: -0.448 ± 0.054 %
|
| 165 |
+
Maximum Δp: 87.240%
|
| 166 |
+
99.9% Δp: 61.174%
|
| 167 |
+
99.0% Δp: 29.777%
|
| 168 |
+
95.0% Δp: 11.711%
|
| 169 |
+
90.0% Δp: 5.997%
|
| 170 |
+
75.0% Δp: 0.739%
|
| 171 |
+
Median Δp: -0.002%
|
| 172 |
+
25.0% Δp: -1.151%
|
| 173 |
+
10.0% Δp: -7.265%
|
| 174 |
+
5.0% Δp: -13.841%
|
| 175 |
+
1.0% Δp: -35.601%
|
| 176 |
+
0.1% Δp: -72.418%
|
| 177 |
+
Minimum Δp: -99.297%
|
| 178 |
+
RMS Δp : 9.753 ± 0.121 %
|
| 179 |
+
Same top p: 84.292 ± 0.201 %
|
| 180 |
+
|
results/kld_q8_0.log
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
0.00.395.802 W load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
|
| 5 |
+
0.02.173.489 I cmn init: llama threadpool init, n_threads = 64
|
| 6 |
+
0.02.290.517 I
|
| 7 |
+
0.02.290.675 I system_info: n_threads = 64 (n_threads_batch = 64) / 80 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
| 8 |
+
0.02.341.634 I kl_divergence: computing over 128 chunks, n_ctx=512, batch_size=2048, n_seq=4
|
| 9 |
+
0.06.994.470 I kl_divergence: 4.65 seconds per pass - ETA 2.47 minutes
|
| 10 |
+
|
| 11 |
+
chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p
|
| 12 |
+
1 7.9784 ± 1.5222 -0.02297 ± 0.00947 0.00565 ± 0.00058 2.327 ± 0.251 % 95.686 ± 1.275 %
|
| 13 |
+
2 12.4056 ± 1.7770 -0.01748 ± 0.00581 0.00504 ± 0.00042 1.926 ± 0.163 % 97.059 ± 0.749 %
|
| 14 |
+
3 11.7712 ± 1.3808 -0.00003 ± 0.00842 0.00606 ± 0.00049 2.176 ± 0.176 % 96.863 ± 0.631 %
|
| 15 |
+
4 12.2673 ± 1.2202 -0.00181 ± 0.00709 0.00655 ± 0.00046 2.082 ± 0.142 % 96.667 ± 0.562 %
|
| 16 |
+
5 12.5480 ± 1.0956 -0.00471 ± 0.00601 0.00737 ± 0.00043 2.066 ± 0.124 % 96.314 ± 0.528 %
|
| 17 |
+
6 13.0267 ± 1.0231 -0.00325 ± 0.00511 0.00666 ± 0.00036 1.956 ± 0.110 % 96.536 ± 0.468 %
|
| 18 |
+
7 13.7466 ± 1.0004 -0.00103 ± 0.00490 0.00768 ± 0.00086 2.392 ± 0.393 % 96.359 ± 0.443 %
|
| 19 |
+
8 14.1331 ± 0.9715 -0.00262 ± 0.00471 0.00825 ± 0.00078 2.501 ± 0.337 % 96.225 ± 0.422 %
|
| 20 |
+
9 14.8952 ± 0.9688 0.00056 ± 0.00428 0.00789 ± 0.00069 2.431 ± 0.308 % 96.078 ± 0.405 %
|
| 21 |
+
10 15.4950 ± 0.9597 -0.00009 ± 0.00422 0.00836 ± 0.00064 2.485 ± 0.285 % 95.843 ± 0.395 %
|
| 22 |
+
11 15.7183 ± 0.9304 0.00040 ± 0.00395 0.00830 ± 0.00061 2.545 ± 0.274 % 95.722 ± 0.382 %
|
| 23 |
+
12 15.9461 ± 0.9109 -0.00193 ± 0.00381 0.00888 ± 0.00060 2.747 ± 0.277 % 95.752 ± 0.365 %
|
| 24 |
+
13 16.0231 ± 0.8749 -0.00123 ± 0.00355 0.00857 ± 0.00056 2.671 ± 0.263 % 95.897 ± 0.345 %
|
| 25 |
+
14 15.7095 ± 0.8261 -0.00197 ± 0.00341 0.00892 ± 0.00053 2.745 ± 0.240 % 95.770 ± 0.337 %
|
| 26 |
+
15 15.3583 ± 0.7830 -0.00136 ± 0.00328 0.00888 ± 0.00050 2.765 ± 0.225 % 95.948 ± 0.319 %
|
| 27 |
+
16 15.8462 ± 0.7875 0.00031 ± 0.00336 0.00954 ± 0.00051 2.830 ± 0.208 % 95.882 ± 0.311 %
|
| 28 |
+
17 14.5451 ± 0.6936 0.00097 ± 0.00319 0.00925 ± 0.00048 2.820 ± 0.198 % 96.055 ± 0.296 %
|
| 29 |
+
18 14.6492 ± 0.6846 -0.00025 ± 0.00314 0.00948 ± 0.00047 2.920 ± 0.190 % 96.122 ± 0.285 %
|
| 30 |
+
19 14.7850 ± 0.6756 -0.00094 ± 0.00307 0.00963 ± 0.00045 2.896 ± 0.182 % 95.996 ± 0.282 %
|
| 31 |
+
20 14.4550 ± 0.6493 -0.00075 ± 0.00308 0.01017 ± 0.00046 2.966 ± 0.173 % 95.961 ± 0.276 %
|
| 32 |
+
21 14.2754 ± 0.6232 -0.00190 ± 0.00303 0.01066 ± 0.00047 3.038 ± 0.172 % 95.892 ± 0.271 %
|
| 33 |
+
22 14.0588 ± 0.5988 -0.00224 ± 0.00293 0.01061 ± 0.00046 3.001 ± 0.166 % 95.758 ± 0.269 %
|
| 34 |
+
23 13.6023 ± 0.5631 -0.00216 ± 0.00283 0.01042 ± 0.00044 2.970 ± 0.161 % 95.737 ± 0.264 %
|
| 35 |
+
24 13.2050 ± 0.5342 -0.00182 ± 0.00281 0.01071 ± 0.00044 3.029 ± 0.156 % 95.686 ± 0.260 %
|
| 36 |
+
25 12.8232 ± 0.5080 -0.00262 ± 0.00277 0.01112 ± 0.00044 3.041 ± 0.149 % 95.624 ± 0.256 %
|
| 37 |
+
26 12.6590 ± 0.4924 -0.00195 ± 0.00273 0.01126 ± 0.00043 3.087 ± 0.147 % 95.596 ± 0.252 %
|
| 38 |
+
27 12.0474 ± 0.4567 -0.00158 ± 0.00267 0.01134 ± 0.00043 3.143 ± 0.145 % 95.628 ± 0.246 %
|
| 39 |
+
28 11.9174 ± 0.4431 -0.00039 ± 0.00268 0.01176 ± 0.00044 3.229 ± 0.141 % 95.616 ± 0.242 %
|
| 40 |
+
29 11.9583 ± 0.4381 0.00126 ± 0.00273 0.01193 ± 0.00043 3.215 ± 0.137 % 95.605 ± 0.238 %
|
| 41 |
+
30 11.7536 ± 0.4214 0.00146 ± 0.00267 0.01187 ± 0.00041 3.227 ± 0.133 % 95.608 ± 0.234 %
|
| 42 |
+
31 11.8914 ± 0.4210 0.00132 ± 0.00263 0.01183 ± 0.00040 3.208 ± 0.130 % 95.610 ± 0.230 %
|
| 43 |
+
32 11.8917 �� 0.4148 0.00034 ± 0.00259 0.01196 ± 0.00039 3.205 ± 0.126 % 95.576 ± 0.228 %
|
| 44 |
+
33 11.9642 ± 0.4117 -0.00045 ± 0.00258 0.01218 ± 0.00039 3.190 ± 0.123 % 95.556 ± 0.225 %
|
| 45 |
+
34 12.2412 ± 0.4192 0.00162 ± 0.00267 0.01248 ± 0.00039 3.215 ± 0.120 % 95.513 ± 0.222 %
|
| 46 |
+
35 12.3363 ± 0.4173 0.00081 ± 0.00263 0.01263 ± 0.00039 3.228 ± 0.118 % 95.406 ± 0.222 %
|
| 47 |
+
36 12.5265 ± 0.4191 0.00104 ± 0.00262 0.01269 ± 0.00038 3.227 ± 0.115 % 95.359 ± 0.220 %
|
| 48 |
+
37 12.6415 ± 0.4163 0.00104 ± 0.00257 0.01256 ± 0.00037 3.203 ± 0.113 % 95.421 ± 0.215 %
|
| 49 |
+
38 12.8136 ± 0.4164 0.00105 ± 0.00254 0.01284 ± 0.00038 3.247 ± 0.113 % 95.377 ± 0.213 %
|
| 50 |
+
39 12.7541 ± 0.4093 0.00067 ± 0.00249 0.01277 ± 0.00037 3.244 ± 0.111 % 95.314 ± 0.212 %
|
| 51 |
+
40 12.6970 ± 0.4031 0.00090 ± 0.00245 0.01262 ± 0.00036 3.225 ± 0.109 % 95.363 ± 0.208 %
|
| 52 |
+
41 12.6791 ± 0.3973 0.00094 ± 0.00240 0.01243 ± 0.00035 3.196 ± 0.107 % 95.390 ± 0.205 %
|
| 53 |
+
42 12.5833 ± 0.3888 0.00061 ± 0.00235 0.01234 ± 0.00035 3.177 ± 0.106 % 95.369 ± 0.203 %
|
| 54 |
+
43 12.7079 ± 0.3884 -0.00079 ± 0.00232 0.01224 ± 0.00034 3.154 ± 0.104 % 95.331 ± 0.201 %
|
| 55 |
+
44 12.6505 ± 0.3821 -0.00098 ± 0.00228 0.01216 ± 0.00033 3.151 ± 0.102 % 95.294 ± 0.200 %
|
| 56 |
+
45 12.5761 ± 0.3748 -0.00123 ± 0.00227 0.01227 ± 0.00033 3.181 ± 0.100 % 95.259 ± 0.198 %
|
| 57 |
+
46 12.7440 ± 0.3766 -0.00187 ± 0.00225 0.01240 ± 0.00034 3.173 ± 0.099 % 95.243 ± 0.197 %
|
| 58 |
+
47 12.6549 ± 0.3701 -0.00201 ± 0.00224 0.01247 ± 0.00033 3.191 ± 0.097 % 95.219 ± 0.195 %
|
| 59 |
+
48 12.8558 ± 0.3727 -0.00209 ± 0.00221 0.01247 ± 0.00033 3.200 ± 0.097 % 95.196 ± 0.193 %
|
| 60 |
+
49 12.8560 ± 0.3682 -0.00246 ± 0.00217 0.01237 ± 0.00033 3.198 ± 0.096 % 95.246 ± 0.190 %
|
| 61 |
+
50 12.8767 ± 0.3653 -0.00200 ± 0.00214 0.01232 ± 0.00032 3.194 ± 0.094 % 95.255 ± 0.188 %
|
| 62 |
+
51 13.0232 ± 0.3666 -0.00232 ± 0.00213 0.01244 ± 0.00032 3.191 ± 0.093 % 95.256 ± 0.186 %
|
| 63 |
+
52 13.0739 ± 0.3650 -0.00204 ± 0.00211 0.01250 ± 0.00032 3.191 ± 0.091 % 95.219 ± 0.185 %
|
| 64 |
+
53 13.0617 ± 0.3610 -0.00203 ± 0.00208 0.01242 ± 0.00031 3.181 ± 0.090 % 95.213 ± 0.184 %
|
| 65 |
+
54 13.0988 ± 0.3586 -0.00213 ± 0.00205 0.01236 ± 0.00031 3.171 ± 0.089 % 95.185 ± 0.182 %
|
| 66 |
+
55 13.2056 ± 0.3584 -0.00249 ± 0.00203 0.01231 ± 0.00030 3.160 ± 0.088 % 95.201 ± 0.180 %
|
| 67 |
+
56 13.2356 ± 0.3555 -0.00219 ± 0.00201 0.01233 ± 0.00030 3.161 ± 0.088 % 95.196 ± 0.179 %
|
| 68 |
+
57 13.2685 ± 0.3531 -0.00200 ± 0.00198 0.01224 ± 0.00030 3.154 ± 0.087 % 95.232 ± 0.177 %
|
| 69 |
+
58 13.2639 ± 0.3494 -0.00197 ± 0.00195 0.01219 ± 0.00029 3.148 ± 0.086 % 95.233 ± 0.175 %
|
| 70 |
+
59 13.3342 ± 0.3485 -0.00217 ± 0.00194 0.01226 ± 0.00029 3.154 ± 0.085 % 95.174 ± 0.175 %
|
| 71 |
+
60 13.4136 ± 0.3476 -0.00198 ± 0.00192 0.01224 ± 0.00029 3.149 ± 0.084 % 95.170 ± 0.173 %
|
| 72 |
+
61 13.5080 ± 0.3473 -0.00144 ± 0.00190 0.01225 ± 0.00028 3.144 ± 0.083 % 95.172 ± 0.172 %
|
| 73 |
+
62 13.5521 ± 0.3455 -0.00185 ± 0.00188 0.01221 ± 0.00028 3.135 ± 0.082 % 95.174 ± 0.170 %
|
| 74 |
+
63 13.6112 ± 0.3448 -0.00212 ± 0.00188 0.01226 ± 0.00028 3.139 ± 0.080 % 95.145 ± 0.170 %
|
| 75 |
+
64 13.7409 ± 0.3456 -0.00232 ± 0.00186 0.01226 ± 0.00027 3.135 ± 0.080 % 95.104 ± 0.169 %
|
| 76 |
+
65 13.8995 ± 0.3476 -0.00205 ± 0.00183 0.01217 ± 0.00027 3.120 ± 0.079 % 95.137 ± 0.167 %
|
| 77 |
+
66 14.0348 ± 0.3483 -0.00157 ± 0.00181 0.01215 ± 0.00027 3.113 ± 0.078 % 95.157 ± 0.165 %
|
| 78 |
+
67 14.1707 ± 0.3498 -0.00091 ± 0.00181 0.01222 ± 0.00027 3.111 ± 0.078 % 95.124 ± 0.165 %
|
| 79 |
+
68 14.2984 ± 0.3507 -0.00095 ± 0.00179 0.01220 ± 0.00026 3.109 ± 0.077 % 95.133 ± 0.163 %
|
| 80 |
+
69 14.2179 ± 0.3456 -0.00109 ± 0.00177 0.01217 ± 0.00027 3.094 ± 0.076 % 95.175 ± 0.162 %
|
| 81 |
+
70 14.2860 ± 0.3449 -0.00088 ± 0.00175 0.01216 ± 0.00027 3.086 ± 0.076 % 95.171 ± 0.160 %
|
| 82 |
+
71 14.3481 ± 0.3441 -0.00161 ± 0.00175 0.01232 ± 0.00027 3.107 ± 0.075 % 95.128 ± 0.160 %
|
| 83 |
+
72 14.4159 ± 0.3436 -0.00195 ± 0.00173 0.01231 ± 0.00026 3.102 ± 0.074 % 95.131 ± 0.159 %
|
| 84 |
+
73 14.5100 ± 0.3440 -0.00174 ± 0.00172 0.01233 ± 0.00026 3.101 ± 0.073 % 95.144 ± 0.158 %
|
| 85 |
+
74 14.4564 ± 0.3400 -0.00161 ± 0.00170 0.01229 ± 0.00026 3.111 ± 0.075 % 95.146 ± 0.156 %
|
| 86 |
+
75 14.5101 ± 0.3394 -0.00162 ± 0.00169 0.01231 ± 0.00026 3.116 ± 0.074 % 95.127 ± 0.156 %
|
| 87 |
+
76 14.4404 ± 0.3354 -0.00153 ± 0.00167 0.01224 ± 0.00026 3.107 ± 0.074 % 95.155 ± 0.154 %
|
| 88 |
+
77 14.3672 ± 0.3314 -0.00117 ± 0.00165 0.01218 ± 0.00025 3.099 ± 0.073 % 95.187 ± 0.153 %
|
| 89 |
+
78 14.3384 ± 0.3286 -0.00109 ± 0.00164 0.01210 ± 0.00025 3.090 ± 0.072 % 95.209 ± 0.151 %
|
| 90 |
+
79 14.3140 ± 0.3259 -0.00010 ± 0.00164 0.01214 ± 0.00025 3.094 ± 0.071 % 95.175 ± 0.151 %
|
| 91 |
+
80 14.1595 ± 0.3204 0.00002 ± 0.00162 0.01208 ± 0.00024 3.090 ± 0.071 % 95.206 ± 0.150 %
|
| 92 |
+
81 14.0902 ± 0.3163 -0.00035 ± 0.00160 0.01202 ± 0.00024 3.081 ± 0.070 % 95.236 ± 0.148 %
|
| 93 |
+
82 14.0078 ± 0.3121 -0.00083 ± 0.00159 0.01198 ± 0.00024 3.080 ± 0.070 % 95.242 ± 0.147 %
|
| 94 |
+
83 13.9801 ± 0.3095 -0.00114 ± 0.00159 0.01207 ± 0.00024 3.084 ± 0.069 % 95.219 ± 0.147 %
|
| 95 |
+
84 14.0095 ± 0.3083 -0.00117 ± 0.00158 0.01207 ± 0.00024 3.092 ± 0.069 % 95.210 ± 0.146 %
|
| 96 |
+
85 14.0504 ± 0.3072 -0.00098 ± 0.00157 0.01216 ± 0.00024 3.089 ± 0.069 % 95.183 ± 0.145 %
|
| 97 |
+
86 14.0262 ± 0.3046 -0.00086 ± 0.00156 0.01213 ± 0.00023 3.083 ± 0.068 % 95.176 ± 0.145 %
|
| 98 |
+
87 14.0537 ± 0.3032 -0.00089 ± 0.00155 0.01208 ± 0.00023 3.070 ± 0.067 % 95.204 ± 0.143 %
|
| 99 |
+
88 14.0293 ± 0.3008 -0.00084 ± 0.00154 0.01205 ± 0.00023 3.064 ± 0.067 % 95.241 ± 0.142 %
|
| 100 |
+
89 14.0540 ± 0.2999 -0.00119 ± 0.00155 0.01217 ± 0.00023 3.082 ± 0.067 % 95.228 ± 0.142 %
|
| 101 |
+
90 14.0498 ± 0.2981 -0.00080 ± 0.00154 0.01217 ± 0.00023 3.090 ± 0.067 % 95.242 ± 0.141 %
|
| 102 |
+
91 13.9604 ± 0.2942 -0.00102 ± 0.00153 0.01217 ± 0.00023 3.087 ± 0.067 % 95.247 ± 0.140 %
|
| 103 |
+
92 13.9122 ± 0.2914 -0.00098 ± 0.00152 0.01216 ± 0.00022 3.088 ± 0.066 % 95.251 ± 0.139 %
|
| 104 |
+
93 13.8821 ± 0.2887 -0.00095 ± 0.00151 0.01210 ± 0.00022 3.083 ± 0.066 % 95.269 ± 0.138 %
|
| 105 |
+
94 13.8370 ± 0.2860 -0.00116 ± 0.00150 0.01215 ± 0.00022 3.079 ± 0.065 % 95.257 ± 0.137 %
|
| 106 |
+
95 13.8671 ± 0.2851 -0.00121 ± 0.00149 0.01209 ± 0.00022 3.074 ± 0.065 % 95.269 ± 0.136 %
|
| 107 |
+
96 13.8617 ± 0.2835 -0.00147 ± 0.00148 0.01216 ± 0.00022 3.080 ± 0.064 % 95.261 ± 0.136 %
|
| 108 |
+
97 13.8577 ± 0.2816 -0.00141 ± 0.00148 0.01222 ± 0.00022 3.079 ± 0.064 % 95.221 ± 0.136 %
|
| 109 |
+
98 13.8707 ± 0.2806 -0.00151 ± 0.00148 0.01224 ± 0.00022 3.078 ± 0.063 % 95.234 ± 0.135 %
|
| 110 |
+
99 13.8162 ± 0.2779 -0.00153 ± 0.00147 0.01220 ± 0.00022 3.071 ± 0.063 % 95.231 ± 0.134 %
|
| 111 |
+
100 13.8257 ± 0.2766 -0.00129 ± 0.00147 0.01225 ± 0.00021 3.071 ± 0.062 % 95.216 ± 0.134 %
|
| 112 |
+
101 13.7916 ± 0.2744 -0.00134 ± 0.00146 0.01226 ± 0.00021 3.078 ± 0.063 % 95.213 ± 0.133 %
|
| 113 |
+
102 13.8312 ± 0.2739 -0.00134 ± 0.00145 0.01226 ± 0.00021 3.079 ± 0.063 % 95.213 ± 0.132 %
|
| 114 |
+
103 13.8718 ± 0.2735 -0.00143 ± 0.00144 0.01220 ± 0.00021 3.068 ± 0.062 % 95.226 ± 0.132 %
|
| 115 |
+
104 13.8638 ± 0.2720 -0.00105 ± 0.00143 0.01239 ± 0.00030 3.066 ± 0.062 % 95.219 ± 0.131 %
|
| 116 |
+
105 13.9646 ± 0.2735 -0.00112 ± 0.00143 0.01247 ± 0.00030 3.068 ± 0.061 % 95.197 ± 0.131 %
|
| 117 |
+
106 13.9281 ± 0.2715 -0.00092 ± 0.00143 0.01259 ± 0.00030 3.078 ± 0.061 % 95.179 ± 0.130 %
|
| 118 |
+
107 13.9285 ± 0.2705 -0.00115 ± 0.00142 0.01260 ± 0.00029 3.083 ± 0.060 % 95.195 ± 0.129 %
|
| 119 |
+
108 13.8918 ± 0.2683 -0.00104 ± 0.00141 0.01253 ± 0.00029 3.075 ± 0.060 % 95.200 ± 0.129 %
|
| 120 |
+
109 13.9151 ± 0.2674 -0.00100 ± 0.00140 0.01246 ± 0.00029 3.064 ± 0.060 % 95.222 ± 0.128 %
|
| 121 |
+
110 13.9847 ± 0.2673 -0.00099 ± 0.00139 0.01240 ± 0.00029 3.054 ± 0.059 % 95.191 ± 0.128 %
|
| 122 |
+
111 14.0521 ± 0.2675 -0.00089 ± 0.00138 0.01237 ± 0.00028 3.052 ± 0.059 % 95.202 ± 0.127 %
|
| 123 |
+
112 14.1677 ± 0.2688 -0.00070 ± 0.00137 0.01234 ± 0.00028 3.043 ± 0.059 % 95.224 ± 0.126 %
|
| 124 |
+
113 14.3090 ± 0.2709 -0.00078 ± 0.00136 0.01230 ± 0.00028 3.034 ± 0.058 % 95.239 ± 0.125 %
|
| 125 |
+
114 14.3166 ± 0.2697 -0.00079 ± 0.00135 0.01227 ± 0.00028 3.029 ± 0.058 % 95.263 ± 0.125 %
|
| 126 |
+
115 14.2292 ± 0.2665 -0.00058 ± 0.00134 0.01223 ± 0.00028 3.024 ± 0.058 % 95.270 ± 0.124 %
|
| 127 |
+
116 14.2939 ± 0.2667 -0.00065 ± 0.00133 0.01215 ± 0.00027 3.013 ± 0.057 % 95.284 ± 0.123 %
|
| 128 |
+
117 14.3398 ± 0.2664 -0.00059 ± 0.00133 0.01215 ± 0.00027 3.011 ± 0.057 % 95.261 ± 0.123 %
|
| 129 |
+
118 14.3267 ± 0.2650 -0.00072 ± 0.00132 0.01213 ± 0.00027 3.007 ± 0.057 % 95.274 ± 0.122 %
|
| 130 |
+
119 14.2918 ± 0.2633 -0.00065 ± 0.00132 0.01215 ± 0.00027 3.008 ± 0.056 % 95.261 ± 0.122 %
|
| 131 |
+
120 14.2470 ± 0.2613 -0.00059 ± 0.00131 0.01210 ± 0.00027 2.999 ± 0.056 % 95.281 ± 0.121 %
|
| 132 |
+
121 14.2083 ± 0.2594 -0.00055 ± 0.00130 0.01205 ± 0.00026 2.993 ± 0.056 % 95.275 ± 0.121 %
|
| 133 |
+
122 14.1930 ± 0.2579 -0.00061 ± 0.00129 0.01200 ± 0.00026 2.985 ± 0.055 % 95.291 ± 0.120 %
|
| 134 |
+
123 14.1900 ± 0.2567 -0.00062 ± 0.00128 0.01196 ± 0.00026 2.977 ± 0.055 % 95.291 ± 0.120 %
|
| 135 |
+
124 14.1725 ± 0.2553 -0.00077 ± 0.00127 0.01192 ± 0.00026 2.970 ± 0.055 % 95.307 ± 0.119 %
|
| 136 |
+
125 14.1266 ± 0.2535 -0.00084 ± 0.00127 0.01189 ± 0.00026 2.967 ± 0.054 % 95.313 ± 0.118 %
|
| 137 |
+
126 14.0629 ± 0.2514 -0.00095 ± 0.00126 0.01190 ± 0.00025 2.968 ± 0.054 % 95.322 ± 0.118 %
|
| 138 |
+
127 14.0137 ± 0.2493 -0.00118 ± 0.00126 0.01188 ± 0.00025 2.965 ± 0.054 % 95.328 ± 0.117 %
|
| 139 |
+
128 13.9843 ± 0.2477 -0.00126 ± 0.00125 0.01184 ± 0.00025 2.960 ± 0.053 % 95.343 ± 0.117 %
|
| 140 |
+
|
| 141 |
+
====== Perplexity statistics ======
|
| 142 |
+
Mean PPL(Q) : 13.984311 ± 0.247651
|
| 143 |
+
Mean PPL(base) : 14.001939 ± 0.247589
|
| 144 |
+
Cor(ln(PPL(Q)), ln(PPL(base))): 99.75%
|
| 145 |
+
Mean ln(PPL(Q)/PPL(base)) : -0.001260 ± 0.001248
|
| 146 |
+
Mean PPL(Q)/PPL(base) : 0.998741 ± 0.001247
|
| 147 |
+
Mean PPL(Q)-PPL(base) : -0.017628 ± 0.017463
|
| 148 |
+
|
| 149 |
+
====== KL divergence statistics ======
|
| 150 |
+
Mean KLD: 0.011835 ± 0.000250
|
| 151 |
+
Maximum KLD: 5.668270
|
| 152 |
+
99.9% KLD: 0.419666
|
| 153 |
+
99.0% KLD: 0.125107
|
| 154 |
+
95.0% KLD: 0.045340
|
| 155 |
+
90.0% KLD: 0.026441
|
| 156 |
+
Median KLD: 0.003751
|
| 157 |
+
10.0% KLD: 0.000044
|
| 158 |
+
5.0% KLD: 0.000006
|
| 159 |
+
1.0% KLD: -0.000000
|
| 160 |
+
0.1% KLD: -0.000017
|
| 161 |
+
Minimum KLD: -0.000174
|
| 162 |
+
|
| 163 |
+
====== Token probability statistics ======
|
| 164 |
+
Mean Δp: 0.005 ± 0.016 %
|
| 165 |
+
Maximum Δp: 45.738%
|
| 166 |
+
99.9% Δp: 22.590%
|
| 167 |
+
99.0% Δp: 9.267%
|
| 168 |
+
95.0% Δp: 3.556%
|
| 169 |
+
90.0% Δp: 1.808%
|
| 170 |
+
75.0% Δp: 0.234%
|
| 171 |
+
Median Δp: 0.000%
|
| 172 |
+
25.0% Δp: -0.246%
|
| 173 |
+
10.0% Δp: -1.743%
|
| 174 |
+
5.0% Δp: -3.565%
|
| 175 |
+
1.0% Δp: -9.279%
|
| 176 |
+
0.1% Δp: -23.139%
|
| 177 |
+
Minimum Δp: -57.307%
|
| 178 |
+
RMS Δp : 2.960 ± 0.053 %
|
| 179 |
+
Same top p: 95.343 ± 0.117 %
|
| 180 |
+
|
results/kld_vq3.log
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
0.00.345.453 W load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
|
| 5 |
+
0.00.841.454 I cmn init: llama threadpool init, n_threads = 64
|
| 6 |
+
0.00.897.064 I
|
| 7 |
+
0.00.897.251 I system_info: n_threads = 64 (n_threads_batch = 64) / 80 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
| 8 |
+
0.00.945.493 I kl_divergence: computing over 128 chunks, n_ctx=512, batch_size=2048, n_seq=4
|
| 9 |
+
0.08.670.934 I kl_divergence: 7.73 seconds per pass - ETA 4.12 minutes
|
| 10 |
+
|
| 11 |
+
chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p
|
| 12 |
+
1 8.7929 ± 1.6858 0.07425 ± 0.04557 0.17091 ± 0.02320 12.032 ± 1.499 % 85.882 ± 2.185 %
|
| 13 |
+
2 13.6217 ± 1.9689 0.07604 ± 0.03397 0.17954 ± 0.01555 11.748 ± 1.083 % 83.137 ± 1.660 %
|
| 14 |
+
3 12.9068 ± 1.5086 0.09207 ± 0.02999 0.20059 ± 0.01444 12.413 ± 0.876 % 83.137 ± 1.355 %
|
| 15 |
+
4 13.5700 ± 1.3523 0.09911 ± 0.02707 0.21126 ± 0.01304 12.263 ± 0.720 % 82.843 ± 1.181 %
|
| 16 |
+
5 13.7100 ± 1.2083 0.08386 ± 0.02508 0.21051 ± 0.01099 11.956 ± 0.619 % 81.490 ± 1.088 %
|
| 17 |
+
6 14.5695 ± 1.1658 0.10868 ± 0.02260 0.20880 ± 0.00953 11.884 ± 0.564 % 80.784 ± 1.008 %
|
| 18 |
+
7 15.0196 ± 1.1103 0.08753 ± 0.02061 0.20157 ± 0.00851 11.722 ± 0.522 % 81.232 ± 0.924 %
|
| 19 |
+
8 15.3338 ± 1.0724 0.07892 ± 0.01934 0.20749 ± 0.00790 11.801 ± 0.480 % 80.392 ± 0.879 %
|
| 20 |
+
9 16.2651 ± 1.0832 0.08854 ± 0.01794 0.20346 ± 0.00726 11.554 ± 0.450 % 80.131 ± 0.833 %
|
| 21 |
+
10 16.7569 ± 1.0578 0.07820 ± 0.01723 0.20808 ± 0.00699 11.526 ± 0.426 % 79.686 ± 0.797 %
|
| 22 |
+
11 16.9390 ± 1.0201 0.07519 ± 0.01632 0.20683 ± 0.00689 11.522 ± 0.403 % 79.822 ± 0.758 %
|
| 23 |
+
12 17.1636 ± 0.9944 0.07165 ± 0.01614 0.22517 ± 0.00774 12.212 ± 0.429 % 79.477 ± 0.730 %
|
| 24 |
+
13 17.1726 ± 0.9497 0.06805 ± 0.01524 0.21845 ± 0.00723 11.994 ± 0.409 % 79.608 ± 0.700 %
|
| 25 |
+
14 16.8770 ± 0.8995 0.06972 ± 0.01486 0.22472 ± 0.00722 12.269 ± 0.396 % 79.552 ± 0.675 %
|
| 26 |
+
15 16.4804 ± 0.8526 0.06915 ± 0.01442 0.22778 ± 0.00706 12.489 ± 0.390 % 79.739 ± 0.650 %
|
| 27 |
+
16 16.8638 ± 0.8467 0.06254 ± 0.01439 0.23343 ± 0.00690 12.753 ± 0.378 % 79.412 ± 0.633 %
|
| 28 |
+
17 15.5087 ± 0.7481 0.06512 ± 0.01373 0.22661 ± 0.00659 12.651 ± 0.368 % 79.862 ± 0.609 %
|
| 29 |
+
18 15.5547 ± 0.7349 0.05972 ± 0.01334 0.22681 ± 0.00645 12.748 ± 0.358 % 80.153 ± 0.589 %
|
| 30 |
+
19 15.6173 ± 0.7194 0.05383 ± 0.01315 0.23196 ± 0.00650 13.001 ± 0.362 % 79.876 ± 0.576 %
|
| 31 |
+
20 15.4209 ± 0.7014 0.06393 ± 0.01303 0.23742 ± 0.00642 13.176 ± 0.353 % 79.882 ± 0.561 %
|
| 32 |
+
21 15.1014 ± 0.6653 0.05436 ± 0.01285 0.24111 ± 0.00642 13.229 ± 0.346 % 79.869 ± 0.548 %
|
| 33 |
+
22 14.8532 ± 0.6376 0.05272 ± 0.01252 0.23807 ± 0.00618 13.089 ± 0.336 % 79.982 ± 0.534 %
|
| 34 |
+
23 14.3827 ± 0.6000 0.05363 ± 0.01233 0.23815 ± 0.00620 13.254 ± 0.340 % 80.051 ± 0.522 %
|
| 35 |
+
24 13.9584 ± 0.5682 0.05366 ± 0.01213 0.24199 ± 0.00611 13.396 ± 0.333 % 80.049 ± 0.511 %
|
| 36 |
+
25 13.6135 ± 0.5440 0.05718 ± 0.01210 0.24843 ± 0.00615 13.583 ± 0.329 % 80.031 ± 0.501 %
|
| 37 |
+
26 13.4733 ± 0.5286 0.06039 ± 0.01201 0.25154 ± 0.00612 13.729 ± 0.329 % 80.075 ± 0.491 %
|
| 38 |
+
27 12.7868 ± 0.4892 0.05797 ± 0.01173 0.25028 ± 0.00599 13.708 ± 0.321 % 80.334 ± 0.479 %
|
| 39 |
+
28 12.6315 ± 0.4740 0.05780 ± 0.01161 0.25595 ± 0.00597 13.898 ± 0.316 % 80.084 ± 0.473 %
|
| 40 |
+
29 12.6185 ± 0.4660 0.05500 ± 0.01147 0.25743 ± 0.00584 13.935 ± 0.309 % 80.000 ± 0.465 %
|
| 41 |
+
30 12.4117 ± 0.4489 0.05594 ± 0.01122 0.25506 ± 0.00571 13.900 ± 0.304 % 80.131 ± 0.456 %
|
| 42 |
+
31 12.5078 ± 0.4460 0.05186 ± 0.01101 0.25498 ± 0.00561 13.881 ± 0.300 % 80.063 ± 0.449 %
|
| 43 |
+
32 12.4712 �� 0.4374 0.04791 ± 0.01089 0.25463 ± 0.00550 13.870 ± 0.295 % 80.037 ± 0.443 %
|
| 44 |
+
33 12.6295 ± 0.4384 0.05366 ± 0.01074 0.25690 ± 0.00550 13.772 ± 0.290 % 80.000 ± 0.436 %
|
| 45 |
+
34 12.8252 ± 0.4414 0.04822 ± 0.01068 0.26154 ± 0.00545 13.841 ± 0.284 % 79.758 ± 0.432 %
|
| 46 |
+
35 12.8881 ± 0.4371 0.04457 ± 0.01056 0.26453 ± 0.00541 14.008 ± 0.280 % 79.630 ± 0.426 %
|
| 47 |
+
36 13.0594 ± 0.4379 0.04270 ± 0.01054 0.27076 ± 0.00544 14.118 ± 0.278 % 79.455 ± 0.422 %
|
| 48 |
+
37 13.2544 ± 0.4386 0.04839 ± 0.01037 0.26968 ± 0.00534 14.054 ± 0.275 % 79.491 ± 0.416 %
|
| 49 |
+
38 13.4175 ± 0.4379 0.04711 ± 0.01026 0.27164 ± 0.00527 14.090 ± 0.271 % 79.319 ± 0.411 %
|
| 50 |
+
39 13.3331 ± 0.4292 0.04507 ± 0.01013 0.27231 ± 0.00523 14.125 ± 0.268 % 79.286 ± 0.406 %
|
| 51 |
+
40 13.2822 ± 0.4231 0.04596 ± 0.00999 0.27088 ± 0.00514 14.088 ± 0.265 % 79.451 ± 0.400 %
|
| 52 |
+
41 13.2789 ± 0.4173 0.04717 ± 0.00981 0.26742 ± 0.00503 13.978 ± 0.261 % 79.560 ± 0.394 %
|
| 53 |
+
42 13.2037 ± 0.4094 0.04873 ± 0.00965 0.26591 ± 0.00493 13.953 ± 0.257 % 79.561 ± 0.390 %
|
| 54 |
+
43 13.3499 ± 0.4095 0.04849 ± 0.00953 0.26540 ± 0.00485 13.943 ± 0.254 % 79.526 ± 0.385 %
|
| 55 |
+
44 13.2897 ± 0.4025 0.04831 ± 0.00940 0.26374 ± 0.00477 13.919 ± 0.251 % 79.572 ± 0.381 %
|
| 56 |
+
45 13.2097 ± 0.3947 0.04792 ± 0.00928 0.26402 ± 0.00470 13.951 ± 0.246 % 79.521 ± 0.377 %
|
| 57 |
+
46 13.3727 ± 0.3961 0.04628 ± 0.00920 0.26387 ± 0.00463 13.942 ± 0.243 % 79.531 ± 0.373 %
|
| 58 |
+
47 13.2571 ± 0.3884 0.04448 ± 0.00915 0.26382 ± 0.00460 13.989 ± 0.240 % 79.516 ± 0.369 %
|
| 59 |
+
48 13.5041 ± 0.3926 0.04711 ± 0.00906 0.26399 ± 0.00453 13.988 ± 0.238 % 79.502 ± 0.365 %
|
| 60 |
+
49 13.5273 ± 0.3886 0.04844 ± 0.00891 0.26112 ± 0.00445 13.907 ± 0.235 % 79.632 ± 0.360 %
|
| 61 |
+
50 13.5432 ± 0.3852 0.04847 ± 0.00884 0.26204 ± 0.00440 13.933 ± 0.232 % 79.631 ± 0.357 %
|
| 62 |
+
51 13.7143 ± 0.3874 0.04939 ± 0.00882 0.26449 ± 0.00438 13.944 ± 0.229 % 79.608 ± 0.353 %
|
| 63 |
+
52 13.7416 ± 0.3849 0.04777 ± 0.00881 0.26594 ± 0.00435 13.981 ± 0.227 % 79.570 ± 0.350 %
|
| 64 |
+
53 13.7842 ± 0.3828 0.05181 ± 0.00873 0.26637 ± 0.00431 13.986 ± 0.225 % 79.534 ± 0.347 %
|
| 65 |
+
54 13.8445 ± 0.3810 0.05324 ± 0.00863 0.26514 ± 0.00427 13.920 ± 0.223 % 79.528 ± 0.344 %
|
| 66 |
+
55 13.9728 ± 0.3815 0.05398 ± 0.00852 0.26432 ± 0.00421 13.890 ± 0.220 % 79.558 ± 0.341 %
|
| 67 |
+
56 13.9910 ± 0.3780 0.05331 ± 0.00841 0.26311 ± 0.00415 13.845 ± 0.218 % 79.545 ± 0.338 %
|
| 68 |
+
57 14.0223 ± 0.3753 0.05325 ± 0.00830 0.26145 ± 0.00409 13.773 ± 0.216 % 79.587 ± 0.334 %
|
| 69 |
+
58 14.0275 ± 0.3718 0.05400 ± 0.00822 0.26154 ± 0.00405 13.812 ± 0.215 % 79.561 ± 0.332 %
|
| 70 |
+
59 14.0978 ± 0.3705 0.05351 ± 0.00816 0.26183 ± 0.00400 13.822 ± 0.213 % 79.482 ± 0.329 %
|
| 71 |
+
60 14.2088 ± 0.3703 0.05561 ± 0.00807 0.26119 ± 0.00395 13.768 ± 0.210 % 79.386 ± 0.327 %
|
| 72 |
+
61 14.3146 ± 0.3705 0.05656 ± 0.00801 0.26212 ± 0.00391 13.762 ± 0.208 % 79.376 ± 0.324 %
|
| 73 |
+
62 14.3945 ± 0.3695 0.05846 ± 0.00796 0.26288 ± 0.00390 13.767 ± 0.207 % 79.355 ± 0.322 %
|
| 74 |
+
63 14.4258 ± 0.3678 0.05601 ± 0.00789 0.26309 ± 0.00386 13.756 ± 0.205 % 79.353 ± 0.319 %
|
| 75 |
+
64 14.5472 ± 0.3679 0.05469 ± 0.00782 0.26270 ± 0.00381 13.726 ± 0.203 % 79.301 ± 0.317 %
|
| 76 |
+
65 14.7084 ± 0.3699 0.05452 ± 0.00776 0.26165 ± 0.00377 13.678 ± 0.201 % 79.270 ± 0.315 %
|
| 77 |
+
66 14.8671 ± 0.3714 0.05604 ± 0.00770 0.26179 ± 0.00373 13.658 ± 0.198 % 79.287 ± 0.312 %
|
| 78 |
+
67 15.0551 ± 0.3744 0.05963 ± 0.00765 0.26252 ± 0.00370 13.695 ± 0.198 % 79.216 ± 0.310 %
|
| 79 |
+
68 15.1368 ± 0.3736 0.05604 ± 0.00760 0.26202 ± 0.00366 13.652 ± 0.196 % 79.204 ± 0.308 %
|
| 80 |
+
69 15.0460 ± 0.3682 0.05552 ± 0.00751 0.26009 ± 0.00361 13.598 ± 0.194 % 79.307 ± 0.305 %
|
| 81 |
+
70 15.0963 ± 0.3667 0.05429 ± 0.00746 0.26054 ± 0.00358 13.601 ± 0.193 % 79.188 ± 0.304 %
|
| 82 |
+
71 15.1601 ± 0.3657 0.05344 ± 0.00743 0.26189 ± 0.00357 13.608 ± 0.192 % 79.144 ± 0.302 %
|
| 83 |
+
72 15.2360 ± 0.3654 0.05338 ± 0.00740 0.26273 ± 0.00356 13.598 ± 0.190 % 79.080 ± 0.300 %
|
| 84 |
+
73 15.3417 ± 0.3661 0.05400 ± 0.00735 0.26303 ± 0.00353 13.595 ± 0.188 % 79.049 ± 0.298 %
|
| 85 |
+
74 15.3028 ± 0.3625 0.05529 ± 0.00730 0.26346 ± 0.00351 13.630 ± 0.187 % 79.089 ± 0.296 %
|
| 86 |
+
75 15.3598 ± 0.3618 0.05529 ± 0.00725 0.26404 ± 0.00349 13.637 ± 0.186 % 79.043 ± 0.294 %
|
| 87 |
+
76 15.3021 ± 0.3580 0.05643 ± 0.00717 0.26286 ± 0.00345 13.606 ± 0.184 % 79.066 ± 0.292 %
|
| 88 |
+
77 15.2352 ± 0.3542 0.05749 ± 0.00712 0.26305 ± 0.00344 13.595 ± 0.184 % 79.124 ± 0.290 %
|
| 89 |
+
78 15.2288 ± 0.3519 0.05916 ± 0.00709 0.26360 ± 0.00343 13.596 ± 0.183 % 79.060 ± 0.289 %
|
| 90 |
+
79 15.1920 ± 0.3488 0.05943 ± 0.00704 0.26306 ± 0.00341 13.589 ± 0.182 % 79.077 ± 0.287 %
|
| 91 |
+
80 15.0102 ± 0.3424 0.05837 ± 0.00700 0.26224 ± 0.00339 13.573 ± 0.181 % 79.147 ± 0.284 %
|
| 92 |
+
81 14.9372 ± 0.3382 0.05803 ± 0.00694 0.26172 ± 0.00336 13.569 ± 0.180 % 79.191 ± 0.282 %
|
| 93 |
+
82 14.8559 ± 0.3340 0.05795 ± 0.00688 0.26065 ± 0.00333 13.563 ± 0.179 % 79.264 ± 0.280 %
|
| 94 |
+
83 14.8115 ± 0.3306 0.05662 ± 0.00683 0.26067 ± 0.00331 13.560 ± 0.178 % 79.268 ± 0.279 %
|
| 95 |
+
84 14.8658 ± 0.3300 0.05816 ± 0.00680 0.26144 ± 0.00331 13.583 ± 0.177 % 79.244 ± 0.277 %
|
| 96 |
+
85 14.9223 ± 0.3292 0.05922 ± 0.00677 0.26238 ± 0.00330 13.605 ± 0.176 % 79.165 ± 0.276 %
|
| 97 |
+
86 14.8710 ± 0.3255 0.05763 ± 0.00672 0.26219 ± 0.00327 13.590 ± 0.174 % 79.161 ± 0.274 %
|
| 98 |
+
87 14.8965 ± 0.3240 0.05735 ± 0.00670 0.26222 ± 0.00325 13.581 ± 0.174 % 79.171 ± 0.273 %
|
| 99 |
+
88 14.8732 ± 0.3214 0.05757 ± 0.00666 0.26194 ± 0.00323 13.575 ± 0.173 % 79.184 ± 0.271 %
|
| 100 |
+
89 14.9157 ± 0.3207 0.05832 ± 0.00663 0.26209 ± 0.00320 13.561 ± 0.172 % 79.202 ± 0.269 %
|
| 101 |
+
90 14.9081 ± 0.3188 0.05850 ± 0.00660 0.26264 ± 0.00319 13.588 ± 0.171 % 79.159 ± 0.268 %
|
| 102 |
+
91 14.8268 ± 0.3150 0.05919 ± 0.00655 0.26241 ± 0.00317 13.594 ± 0.170 % 79.186 ± 0.267 %
|
| 103 |
+
92 14.7959 ± 0.3124 0.06060 ± 0.00652 0.26242 ± 0.00316 13.605 ± 0.169 % 79.190 ± 0.265 %
|
| 104 |
+
93 14.7643 ± 0.3096 0.06067 ± 0.00646 0.26115 ± 0.00313 13.586 ± 0.168 % 79.228 ± 0.263 %
|
| 105 |
+
94 14.7099 ± 0.3066 0.06001 ± 0.00642 0.26070 ± 0.00310 13.559 ± 0.167 % 79.241 ± 0.262 %
|
| 106 |
+
95 14.7260 ± 0.3052 0.05889 ± 0.00637 0.25927 ± 0.00307 13.520 ± 0.166 % 79.315 ± 0.260 %
|
| 107 |
+
96 14.7267 ± 0.3034 0.05906 ± 0.00634 0.25956 ± 0.00306 13.544 ± 0.165 % 79.342 ± 0.259 %
|
| 108 |
+
97 14.7503 ± 0.3021 0.06101 ± 0.00631 0.25944 ± 0.00303 13.518 ± 0.164 % 79.325 ± 0.258 %
|
| 109 |
+
98 14.7582 ± 0.3008 0.06051 ± 0.00629 0.25931 ± 0.00302 13.523 ± 0.163 % 79.380 ± 0.256 %
|
| 110 |
+
99 14.6889 ± 0.2976 0.05972 ± 0.00624 0.25856 ± 0.00299 13.510 ± 0.163 % 79.426 ± 0.254 %
|
| 111 |
+
100 14.6781 ± 0.2957 0.05854 ± 0.00620 0.25857 ± 0.00298 13.505 ± 0.162 % 79.404 ± 0.253 %
|
| 112 |
+
101 14.6457 ± 0.2935 0.05875 ± 0.00617 0.25898 ± 0.00296 13.509 ± 0.161 % 79.383 ± 0.252 %
|
| 113 |
+
102 14.6859 ± 0.2930 0.05863 ± 0.00614 0.25847 ± 0.00295 13.490 ± 0.160 % 79.389 ± 0.251 %
|
| 114 |
+
103 14.7290 ± 0.2926 0.05853 ± 0.00610 0.25742 ± 0.00292 13.452 ± 0.159 % 79.417 ± 0.249 %
|
| 115 |
+
104 14.7364 ± 0.2915 0.05999 ± 0.00608 0.25846 ± 0.00294 13.452 ± 0.158 % 79.382 ± 0.248 %
|
| 116 |
+
105 14.8379 ± 0.2929 0.05954 ± 0.00609 0.26076 ± 0.00295 13.476 ± 0.158 % 79.328 ± 0.247 %
|
| 117 |
+
106 14.7710 ± 0.2900 0.05784 ± 0.00605 0.26048 ± 0.00293 13.474 ± 0.156 % 79.375 ± 0.246 %
|
| 118 |
+
107 14.8002 ± 0.2896 0.05955 ± 0.00605 0.26164 ± 0.00294 13.508 ± 0.156 % 79.392 ± 0.245 %
|
| 119 |
+
108 14.7649 ± 0.2874 0.05991 ± 0.00602 0.26062 ± 0.00292 13.494 ± 0.155 % 79.412 ± 0.244 %
|
| 120 |
+
109 14.8008 ± 0.2867 0.06071 ± 0.00598 0.25968 ± 0.00290 13.477 ± 0.154 % 79.439 ± 0.242 %
|
| 121 |
+
110 14.8906 ± 0.2870 0.06177 ± 0.00594 0.25917 ± 0.00290 13.457 ± 0.154 % 79.455 ± 0.241 %
|
| 122 |
+
111 14.9657 ± 0.2873 0.06210 ± 0.00591 0.25979 ± 0.00294 13.446 ± 0.153 % 79.445 ± 0.240 %
|
| 123 |
+
112 15.1198 ± 0.2895 0.06435 ± 0.00588 0.26017 ± 0.00292 13.436 ± 0.152 % 79.408 ± 0.239 %
|
| 124 |
+
113 15.2687 ± 0.2917 0.06414 ± 0.00584 0.25945 ± 0.00290 13.412 ± 0.151 % 79.431 ± 0.238 %
|
| 125 |
+
114 15.2905 ± 0.2906 0.06501 ± 0.00581 0.25944 ± 0.00289 13.420 ± 0.151 % 79.395 ± 0.237 %
|
| 126 |
+
115 15.1885 ± 0.2871 0.06466 ± 0.00578 0.25927 ± 0.00288 13.431 ± 0.150 % 79.427 ± 0.236 %
|
| 127 |
+
116 15.2599 ± 0.2874 0.06474 ± 0.00574 0.25797 ± 0.00286 13.390 ± 0.149 % 79.466 ± 0.235 %
|
| 128 |
+
117 15.3055 ± 0.2870 0.06458 ± 0.00572 0.25771 ± 0.00284 13.382 ± 0.149 % 79.477 ± 0.234 %
|
| 129 |
+
118 15.2849 ± 0.2853 0.06402 ± 0.00569 0.25724 ± 0.00282 13.368 ± 0.148 % 79.472 ± 0.233 %
|
| 130 |
+
119 15.2515 ± 0.2836 0.06434 ± 0.00566 0.25723 ± 0.00281 13.356 ± 0.147 % 79.440 ± 0.232 %
|
| 131 |
+
120 15.1969 ± 0.2814 0.06395 ± 0.00562 0.25657 ± 0.00279 13.343 ± 0.147 % 79.500 ± 0.231 %
|
| 132 |
+
121 15.1514 ± 0.2792 0.06372 ± 0.00560 0.25604 ± 0.00278 13.328 ± 0.146 % 79.546 ± 0.230 %
|
| 133 |
+
122 15.1257 ± 0.2775 0.06304 ± 0.00557 0.25626 ± 0.00277 13.322 ± 0.146 % 79.569 ± 0.229 %
|
| 134 |
+
123 15.1242 ± 0.2762 0.06314 ± 0.00554 0.25607 ± 0.00276 13.305 ± 0.145 % 79.560 ± 0.228 %
|
| 135 |
+
124 15.1094 ± 0.2749 0.06324 ± 0.00551 0.25517 ± 0.00274 13.284 ± 0.145 % 79.620 ± 0.227 %
|
| 136 |
+
125 15.0663 ± 0.2731 0.06356 ± 0.00549 0.25550 ± 0.00273 13.306 ± 0.144 % 79.620 ± 0.226 %
|
| 137 |
+
126 14.9979 ± 0.2709 0.06342 ± 0.00549 0.25608 ± 0.00274 13.335 ± 0.144 % 79.623 ± 0.225 %
|
| 138 |
+
127 14.9375 ± 0.2685 0.06266 ± 0.00545 0.25562 ± 0.00272 13.315 ± 0.144 % 79.642 ± 0.224 %
|
| 139 |
+
128 14.8934 ± 0.2664 0.06173 ± 0.00543 0.25514 ± 0.00271 13.322 ± 0.143 % 79.672 ± 0.223 %
|
| 140 |
+
|
| 141 |
+
====== Perplexity statistics ======
|
| 142 |
+
Mean PPL(Q) : 14.893449 ± 0.266399
|
| 143 |
+
Mean PPL(base) : 14.001939 ± 0.247589
|
| 144 |
+
Cor(ln(PPL(Q)), ln(PPL(base))): 95.35%
|
| 145 |
+
Mean ln(PPL(Q)/PPL(base)) : 0.061726 ± 0.005427
|
| 146 |
+
Mean PPL(Q)/PPL(base) : 1.063670 ± 0.005773
|
| 147 |
+
Mean PPL(Q)-PPL(base) : 0.891510 ± 0.080545
|
| 148 |
+
|
| 149 |
+
====== KL divergence statistics ======
|
| 150 |
+
Mean KLD: 0.255136 ± 0.002708
|
| 151 |
+
Maximum KLD: 11.581626
|
| 152 |
+
99.9% KLD: 5.487684
|
| 153 |
+
99.0% KLD: 2.336201
|
| 154 |
+
95.0% KLD: 1.029843
|
| 155 |
+
90.0% KLD: 0.625292
|
| 156 |
+
Median KLD: 0.103638
|
| 157 |
+
10.0% KLD: 0.001261
|
| 158 |
+
5.0% KLD: 0.000222
|
| 159 |
+
1.0% KLD: 0.000012
|
| 160 |
+
0.1% KLD: 0.000000
|
| 161 |
+
Minimum KLD: -0.000018
|
| 162 |
+
|
| 163 |
+
====== Token probability statistics ======
|
| 164 |
+
Mean Δp: -0.685 ± 0.074 %
|
| 165 |
+
Maximum Δp: 99.720%
|
| 166 |
+
99.9% Δp: 78.922%
|
| 167 |
+
99.0% Δp: 41.064%
|
| 168 |
+
95.0% Δp: 16.688%
|
| 169 |
+
90.0% Δp: 8.585%
|
| 170 |
+
75.0% Δp: 0.999%
|
| 171 |
+
Median Δp: -0.003%
|
| 172 |
+
25.0% Δp: -1.657%
|
| 173 |
+
10.0% Δp: -10.508%
|
| 174 |
+
5.0% Δp: -20.428%
|
| 175 |
+
1.0% Δp: -51.162%
|
| 176 |
+
0.1% Δp: -87.436%
|
| 177 |
+
Minimum Δp: -99.001%
|
| 178 |
+
RMS Δp : 13.322 ± 0.143 %
|
| 179 |
+
Same top p: 79.672 ± 0.223 %
|
| 180 |
+
|
results/ppl_base_q8dq6.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
0.00.332.478 W load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
|
| 5 |
+
0.01.021.001 I cmn init: llama threadpool init, n_threads = 64
|
| 6 |
+
0.01.083.634 I
|
| 7 |
+
0.01.083.774 I system_info: n_threads = 64 (n_threads_batch = 64) / 80 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
| 8 |
+
0.01.083.779 I perplexity: tokenizing the input ..
|
| 9 |
+
0.01.661.434 I perplexity: tokenization took 577.643 ms
|
| 10 |
+
0.01.662.296 I perplexity: calculating perplexity over 256 chunks, n_ctx=512, batch_size=2048, n_seq=4
|
| 11 |
+
0.07.520.957 I perplexity: 5.86 seconds per pass - ETA 6.23 minutes
|
| 12 |
+
[1]8.1760,[2]12.5990,[3]11.8685,[4]12.3912,[5]12.6999,[6]13.1507,[7]13.8532,[8]14.2408,[9]15.0203,[10]15.5944,[11]15.8327,[12]16.0859,[13]16.1902,[14]15.8492,[15]15.4813,[16]15.9741,[17]14.6659,[18]14.7784,[19]14.9111,[20]14.5869,[21]14.3900,[22]14.1716,[23]13.6994,[24]13.2975,[25]12.9230,[26]12.7526,[27]12.1262,[28]11.9874,[29]12.0150,[30]11.7953,[31]11.9487,[32]11.9482,[33]12.0238,[34]12.2895,[35]12.3914,[36]12.5866,[37]12.7177,[38]12.8833,[39]12.8102,[40]12.7511,[41]12.7324,[42]12.6439,[43]12.7793,[44]12.7220,[45]12.6415,[46]12.8120,[47]12.7284,[48]12.9300,[49]12.9314,[50]12.9537,[51]13.1195,[52]13.1675,[53]13.1573,[54]13.1967,[55]13.3013,[56]13.3306,[57]13.3631,[58]13.3592,[59]13.4359,[60]13.5147,[61]13.6081,[62]13.6494,[63]13.7118,[64]13.8436,[65]13.9977,[66]14.1275,[67]14.2674,[68]14.3898,[69]14.3072,[70]14.3712,[71]14.4418,[72]14.5141,[73]14.6097,[74]14.5566,[75]14.6104,[76]14.5355,[77]14.4600,[78]14.4290,[79]14.4031,[80]14.2481,[81]14.1859,[82]14.1022,[83]14.0785,[84]14.1117,[85]14.1530,[86]14.1228,[87]14.1516,[88]14.1239,[89]14.1581,[90]14.1505,[91]14.0643,[92]14.0201,[93]13.9880,[94]13.9410,[95]13.9709,[96]13.9707,[97]13.9618,[98]13.9791,[99]13.9227,[100]13.9290,[101]13.8896,[102]13.9313,[103]13.9681,[104]13.9554,[105]14.0603,[106]14.0206,[107]14.0305,[108]13.9946,[109]14.0181,[110]14.0882,[111]14.1523,[112]14.2709,[113]14.4146,[114]14.4226,[115]14.3334,[116]14.3994,[117]14.4436,[118]14.4301,[119]14.3979,[120]14.3522,[121]14.3122,[122]14.2960,[123]14.2923,[124]14.2747,[125]14.2259,[126]14.1607,[127]14.1116,[128]14.0805,[129]14.0023,[130]13.9695,[131]13.8749,[132]13.7912,[133]13.7482,[134]13.6936,[135]13.6947,[136]13.6748,[137]13.6642,[138]13.6795,[139]13.6380,[140]13.5704,[141]13.5444,[142]13.5407,[143]13.5737,[144]13.6246,[145]13.5630,[146]13.5407,[147]13.5671,[148]13.5432,[149]13.4906,[150]13.4550,[151]13.3817,[152]13.2936,[153]13.1719,[154]13.0969,[155]13.0686,[156]13.0415,[157]13.0160,[158]12.9986,[159]12.9263,[160]12.8801,[161]12.8286,[162]12.7944,[163]12.7353,[164]12.7177,[165]12.7334,[166]12.7224,[167]12.7565,[168]12.7193,[169]12.7087,[170]12.6797,[171]12.7366,[172]12.7362,[173]12.7761,[174]12.7837,[175]12.8370,[176]12.9055,[177]12.9558,[178]13.0296,[179]13.0858,[180]13.1720,[181]13.2387,[182]13.2529,[183]13.2626,[184]13.2658,[185]13.3166,[186]13.3406,[187]13.3668,[188]13.3955,[189]13.4089,[190]13.4515,[191]13.4566,[192]13.4645,[193]13.4890,[194]13.5215,[195]13.5562,[196]13.5410,[197]13.6011,[198]13.6423,[199]13.7049,[200]13.7226,[201]13.7280,[202]13.7055,[203]13.6803,[204]13.6645,[205]13.6270,[206]13.6676,[207]13.6859,[208]13.6783,[209]13.6410,[210]13.6084,[211]13.5767,[212]13.5214,[213]13.5673,[214]13.5543,[215]13.5470,[216]13.5229,[217]13.5244,[218]13.4838,[219]13.5124,[220]13.4765,[221]13.4496,[222]13.4267,[223]13.3926,[224]13.3385,[225]13.3241,[226]13.3235,[227]13.3049,[228]13.3171,[229]13.2667,[230]13.2306,[231]13.2162,[232]13.1865,[233]13.1506,[234]13.1266,[235]13.0851,[236]13.0714,[237]13.0662,[238]13.0757,[239]13.0525,[240]13.0232,[241]13.0179,[242]12.9761,[243]12.9369,[244]12.9228,[245]12.9110,[246]12.9175,[247]12.9193,[248]12.9447,[249]12.9679,[250]12.9919,[251]13.0153,[252]13.0342,[253]13.0694,[254]13.0964,[255]13.1149,[256]13.1406,
|
| 13 |
+
4.42.811.834 I Final estimate: PPL = 13.1406 +/- 0.16370
|
| 14 |
+
|
results/ppl_bf16.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
0.00.664.154 W load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
|
| 5 |
+
0.01.816.200 I cmn init: llama threadpool init, n_threads = 64
|
| 6 |
+
0.01.894.167 I
|
| 7 |
+
0.01.894.333 I system_info: n_threads = 64 (n_threads_batch = 64) / 80 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
| 8 |
+
0.01.894.339 I perplexity: tokenizing the input ..
|
| 9 |
+
0.02.484.198 I perplexity: tokenization took 589.852 ms
|
| 10 |
+
0.02.484.413 I perplexity: calculating perplexity over 256 chunks, n_ctx=512, batch_size=2048, n_seq=4
|
| 11 |
+
0.08.139.504 I perplexity: 5.66 seconds per pass - ETA 6.02 minutes
|
| 12 |
+
[1]8.1805,[2]12.6373,[3]11.8638,[4]12.3617,[5]12.6663,[6]13.1201,[7]13.8095,[8]14.2267,[9]14.9397,[10]15.5506,[11]15.7619,[12]16.0234,[13]16.0860,[14]15.7798,[15]15.4154,[16]15.8763,[17]14.5611,[18]14.6866,[19]14.8310,[20]14.5036,[21]14.3380,[22]14.1237,[23]13.6626,[24]13.2579,[25]12.8837,[26]12.7161,[27]12.0963,[28]11.9504,[29]11.9826,[30]11.7738,[31]11.9129,[32]11.9238,[33]12.0048,[34]12.2665,[35]12.3705,[36]12.5638,[37]12.6779,[38]12.8489,[39]12.7929,[40]12.7321,[41]12.7124,[42]12.6202,[43]12.7620,[44]12.7058,[45]12.6333,[46]12.8092,[47]12.7222,[48]12.9243,[49]12.9285,[50]12.9425,[51]13.0944,[52]13.1409,[53]13.1278,[54]13.1657,[55]13.2771,[56]13.3026,[57]13.3324,[58]13.3268,[59]13.4020,[60]13.4787,[61]13.5656,[62]13.6147,[63]13.6801,[64]13.8127,[65]13.9679,[66]14.0966,[67]14.2232,[68]14.3513,[69]14.2719,[70]14.3387,[71]14.4117,[72]14.4844,[73]14.5771,[74]14.5208,[75]14.5749,[76]14.5037,[77]14.4244,[78]14.3939,[79]14.3547,[80]14.1975,[81]14.1328,[82]14.0565,[83]14.0326,[84]14.0621,[85]14.1000,[86]14.0736,[87]14.1024,[88]14.0768,[89]14.1085,[90]14.0988,[91]14.0127,[92]13.9633,[93]13.9322,[94]13.8896,[95]13.9200,[96]13.9180,[97]13.9126,[98]13.9287,[99]13.8738,[100]13.8797,[101]13.8458,[102]13.8852,[103]13.9268,[104]13.9133,[105]14.0168,[106]13.9770,[107]13.9804,[108]13.9423,[109]13.9648,[110]14.0343,[111]14.1001,[112]14.2130,[113]14.3565,[114]14.3641,[115]14.2734,[116]14.3390,[117]14.3854,[118]14.3741,[119]14.3383,[120]14.2921,[121]14.2524,[122]14.2377,[123]14.2355,[124]14.2198,[125]14.1748,[126]14.1122,[127]14.0661,[128]14.0375,[129]13.9594,[130]13.9280,[131]13.8358,[132]13.7499,[133]13.7100,[134]13.6526,[135]13.6558,[136]13.6365,[137]13.6264,[138]13.6470,[139]13.6043,[140]13.5353,[141]13.5099,[142]13.5059,[143]13.5384,[144]13.5885,[145]13.5278,[146]13.5045,[147]13.5320,[148]13.5050,[149]13.4545,[150]13.4183,[151]13.3456,[152]13.2589,[153]13.1355,[154]13.0601,[155]13.0301,[156]13.0024,[157]12.9786,[158]12.9638,[159]12.8907,[160]12.8439,[161]12.7937,[162]12.7580,[163]12.7007,[164]12.6817,[165]12.6978,[166]12.6831,[167]12.7132,[168]12.6732,[169]12.6602,[170]12.6311,[171]12.6869,[172]12.6840,[173]12.7225,[174]12.7290,[175]12.7811,[176]12.8497,[177]12.9020,[178]12.9755,[179]13.0315,[180]13.1156,[181]13.1804,[182]13.1945,[183]13.2054,[184]13.2098,[185]13.2606,[186]13.2846,[187]13.3110,[188]13.3354,[189]13.3507,[190]13.3913,[191]13.3976,[192]13.4061,[193]13.4302,[194]13.4633,[195]13.4982,[196]13.4835,[197]13.5450,[198]13.5842,[199]13.6452,[200]13.6640,[201]13.6694,[202]13.6467,[203]13.6197,[204]13.6055,[205]13.5675,[206]13.6075,[207]13.6271,[208]13.6196,[209]13.5835,[210]13.5512,[211]13.5212,[212]13.4651,[213]13.5100,[214]13.4971,[215]13.4926,[216]13.4686,[217]13.4702,[218]13.4294,[219]13.4584,[220]13.4241,[221]13.3952,[222]13.3728,[223]13.3390,[224]13.2861,[225]13.2715,[226]13.2720,[227]13.2543,[228]13.2675,[229]13.2162,[230]13.1823,[231]13.1685,[232]13.1378,[233]13.1039,[234]13.0780,[235]13.0370,[236]13.0236,[237]13.0204,[238]13.0306,[239]13.0077,[240]12.9768,[241]12.9706,[242]12.9294,[243]12.8924,[244]12.8791,[245]12.8673,[246]12.8742,[247]12.8761,[248]12.9026,[249]12.9283,[250]12.9527,[251]12.9748,[252]12.9937,[253]13.0298,[254]13.0568,[255]13.0768,[256]13.1045,
|
| 13 |
+
4.56.999.924 I Final estimate: PPL = 13.1045 +/- 0.16316
|
| 14 |
+
|
results/ppl_q4_k_m.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
0.00.337.526 W load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
|
| 5 |
+
0.05.270.470 I cmn init: llama threadpool init, n_threads = 64
|
| 6 |
+
0.05.412.905 I
|
| 7 |
+
0.05.413.028 I system_info: n_threads = 64 (n_threads_batch = 64) / 80 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
| 8 |
+
0.05.413.031 I perplexity: tokenizing the input ..
|
| 9 |
+
0.06.002.832 I perplexity: tokenization took 589.791 ms
|
| 10 |
+
0.06.003.043 I perplexity: calculating perplexity over 256 chunks, n_ctx=512, batch_size=2048, n_seq=4
|
| 11 |
+
0.11.977.471 I perplexity: 5.97 seconds per pass - ETA 6.37 minutes
|
| 12 |
+
[1]9.2135,[2]13.7032,[3]12.8484,[4]13.3780,[5]13.5508,[6]13.7879,[7]14.6299,[8]14.9460,[9]15.4850,[10]16.0452,[11]16.2375,[12]16.4255,[13]16.5657,[14]16.1432,[15]15.7884,[16]16.2771,[17]14.8911,[18]14.9903,[19]15.1307,[20]14.7837,[21]14.5570,[22]14.3470,[23]13.9338,[24]13.5553,[25]13.2813,[26]13.0897,[27]12.4494,[28]12.3204,[29]12.3959,[30]12.2044,[31]12.3413,[32]12.3383,[33]12.4370,[34]12.6489,[35]12.7494,[36]12.9727,[37]13.0931,[38]13.3507,[39]13.2475,[40]13.1842,[41]13.1719,[42]13.0768,[43]13.2066,[44]13.1273,[45]13.0755,[46]13.2625,[47]13.1535,[48]13.3759,[49]13.3656,[50]13.3755,[51]13.5047,[52]13.5339,[53]13.5469,[54]13.5795,[55]13.7141,[56]13.7510,[57]13.7730,[58]13.7705,[59]13.8400,[60]13.9250,[61]14.0361,[62]14.0615,[63]14.1002,[64]14.2422,[65]14.3915,[66]14.5562,[67]14.7186,[68]14.8362,[69]14.7468,[70]14.7955,[71]14.9094,[72]14.9684,[73]15.0539,[74]15.0111,[75]15.0635,[76]15.0026,[77]14.9241,[78]14.8713,[79]14.8516,[80]14.6729,[81]14.5973,[82]14.4948,[83]14.4599,[84]14.4753,[85]14.5235,[86]14.5066,[87]14.5267,[88]14.5015,[89]14.5328,[90]14.5096,[91]14.4196,[92]14.3630,[93]14.3360,[94]14.2916,[95]14.3150,[96]14.3164,[97]14.3207,[98]14.3503,[99]14.2947,[100]14.2994,[101]14.2694,[102]14.2993,[103]14.3255,[104]14.3197,[105]14.4117,[106]14.3633,[107]14.3511,[108]14.3071,[109]14.3298,[110]14.4074,[111]14.4750,[112]14.5840,[113]14.7311,[114]14.7301,[115]14.6281,[116]14.6985,[117]14.7350,[118]14.7223,[119]14.6785,[120]14.6353,[121]14.5970,[122]14.5769,[123]14.5777,[124]14.5521,[125]14.5122,[126]14.4496,[127]14.3935,[128]14.3726,[129]14.2937,[130]14.2677,[131]14.1732,[132]14.0984,[133]14.0409,[134]13.9755,[135]13.9755,[136]13.9515,[137]13.9431,[138]13.9528,[139]13.9144,[140]13.8557,[141]13.8325,[142]13.8238,[143]13.8542,[144]13.9010,[145]13.8524,[146]13.8356,[147]13.8569,[148]13.8378,[149]13.7917,[150]13.7522,[151]13.6811,[152]13.5931,[153]13.4729,[154]13.3979,[155]13.3660,[156]13.3360,[157]13.3120,[158]13.2969,[159]13.2213,[160]13.1668,[161]13.1178,[162]13.0789,[163]13.0359,[164]13.0116,[165]13.0230,[166]13.0086,[167]13.0395,[168]12.9959,[169]12.9847,[170]12.9523,[171]13.0060,[172]12.9974,[173]13.0343,[174]13.0484,[175]13.1002,[176]13.1657,[177]13.2156,[178]13.2921,[179]13.3450,[180]13.4291,[181]13.4945,[182]13.5009,[183]13.5063,[184]13.5073,[185]13.5709,[186]13.6093,[187]13.6405,[188]13.6667,[189]13.6735,[190]13.7142,[191]13.7229,[192]13.7306,[193]13.7575,[194]13.7871,[195]13.8295,[196]13.8210,[197]13.8804,[198]13.9249,[199]13.9885,[200]14.0141,[201]14.0222,[202]14.0030,[203]13.9746,[204]13.9546,[205]13.9140,[206]13.9484,[207]13.9722,[208]13.9640,[209]13.9235,[210]13.8870,[211]13.8508,[212]13.7936,[213]13.8296,[214]13.8202,[215]13.8084,[216]13.7839,[217]13.7873,[218]13.7418,[219]13.7585,[220]13.7230,[221]13.6956,[222]13.6729,[223]13.6358,[224]13.5863,[225]13.5701,[226]13.5659,[227]13.5513,[228]13.5685,[229]13.5197,[230]13.4902,[231]13.4738,[232]13.4418,[233]13.4086,[234]13.3935,[235]13.3467,[236]13.3312,[237]13.3225,[238]13.3322,[239]13.3140,[240]13.2916,[241]13.2819,[242]13.2367,[243]13.2011,[244]13.1893,[245]13.1759,[246]13.1821,[247]13.1875,[248]13.2125,[249]13.2407,[250]13.2624,[251]13.2897,[252]13.3130,[253]13.3566,[254]13.3796,[255]13.3960,[256]13.4217,
|
| 13 |
+
5.01.087.519 I Final estimate: PPL = 13.4217 +/- 0.16689
|
| 14 |
+
|
results/ppl_q8_0.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
0.00.325.793 W load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
|
| 5 |
+
0.01.057.881 I cmn init: llama threadpool init, n_threads = 64
|
| 6 |
+
0.01.130.078 I
|
| 7 |
+
0.01.130.261 I system_info: n_threads = 64 (n_threads_batch = 64) / 80 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
| 8 |
+
0.01.130.269 I perplexity: tokenizing the input ..
|
| 9 |
+
0.01.701.125 I perplexity: tokenization took 570.846 ms
|
| 10 |
+
0.01.701.332 I perplexity: calculating perplexity over 256 chunks, n_ctx=512, batch_size=2048, n_seq=4
|
| 11 |
+
0.06.959.871 I perplexity: 5.26 seconds per pass - ETA 5.60 minutes
|
| 12 |
+
[1]7.9784,[2]12.4056,[3]11.7712,[4]12.2673,[5]12.5480,[6]13.0267,[7]13.7466,[8]14.1331,[9]14.8952,[10]15.4950,[11]15.7183,[12]15.9461,[13]16.0231,[14]15.7095,[15]15.3583,[16]15.8462,[17]14.5451,[18]14.6492,[19]14.7850,[20]14.4550,[21]14.2754,[22]14.0588,[23]13.6023,[24]13.2050,[25]12.8232,[26]12.6590,[27]12.0474,[28]11.9174,[29]11.9583,[30]11.7536,[31]11.8914,[32]11.8917,[33]11.9642,[34]12.2412,[35]12.3363,[36]12.5265,[37]12.6415,[38]12.8136,[39]12.7541,[40]12.6970,[41]12.6791,[42]12.5833,[43]12.7079,[44]12.6505,[45]12.5761,[46]12.7440,[47]12.6549,[48]12.8558,[49]12.8560,[50]12.8767,[51]13.0232,[52]13.0739,[53]13.0617,[54]13.0988,[55]13.2056,[56]13.2356,[57]13.2685,[58]13.2639,[59]13.3342,[60]13.4136,[61]13.5080,[62]13.5521,[63]13.6112,[64]13.7409,[65]13.8995,[66]14.0348,[67]14.1707,[68]14.2984,[69]14.2179,[70]14.2860,[71]14.3481,[72]14.4159,[73]14.5100,[74]14.4564,[75]14.5101,[76]14.4404,[77]14.3672,[78]14.3384,[79]14.3140,[80]14.1595,[81]14.0902,[82]14.0078,[83]13.9801,[84]14.0095,[85]14.0504,[86]14.0262,[87]14.0537,[88]14.0293,[89]14.0540,[90]14.0498,[91]13.9604,[92]13.9122,[93]13.8821,[94]13.8370,[95]13.8671,[96]13.8617,[97]13.8577,[98]13.8707,[99]13.8162,[100]13.8257,[101]13.7916,[102]13.8312,[103]13.8718,[104]13.8638,[105]13.9646,[106]13.9281,[107]13.9285,[108]13.8918,[109]13.9151,[110]13.9847,[111]14.0521,[112]14.1677,[113]14.3090,[114]14.3166,[115]14.2292,[116]14.2939,[117]14.3398,[118]14.3267,[119]14.2918,[120]14.2470,[121]14.2083,[122]14.1930,[123]14.1900,[124]14.1725,[125]14.1266,[126]14.0629,[127]14.0137,[128]13.9843,[129]13.9073,[130]13.8775,[131]13.7865,[132]13.7023,[133]13.6629,[134]13.6036,[135]13.6047,[136]13.5847,[137]13.5762,[138]13.5921,[139]13.5514,[140]13.4831,[141]13.4580,[142]13.4551,[143]13.4916,[144]13.5402,[145]13.4803,[146]13.4558,[147]13.4815,[148]13.4584,[149]13.4068,[150]13.3693,[151]13.2964,[152]13.2096,[153]13.0872,[154]13.0113,[155]12.9817,[156]12.9544,[157]12.9313,[158]12.9158,[159]12.8431,[160]12.7976,[161]12.7470,[162]12.7129,[163]12.6554,[164]12.6374,[165]12.6513,[166]12.6387,[167]12.6710,[168]12.6326,[169]12.6212,[170]12.5928,[171]12.6498,[172]12.6483,[173]12.6881,[174]12.6938,[175]12.7468,[176]12.8162,[177]12.8680,[178]12.9423,[179]12.9975,[180]13.0810,[181]13.1472,[182]13.1630,[183]13.1737,[184]13.1763,[185]13.2267,[186]13.2500,[187]13.2761,[188]13.3031,[189]13.3187,[190]13.3600,[191]13.3656,[192]13.3729,[193]13.3977,[194]13.4303,[195]13.4644,[196]13.4485,[197]13.5105,[198]13.5510,[199]13.6128,[200]13.6309,[201]13.6364,[202]13.6143,[203]13.5870,[204]13.5725,[205]13.5357,[206]13.5772,[207]13.5967,[208]13.5893,[209]13.5525,[210]13.5200,[211]13.4895,[212]13.4357,[213]13.4793,[214]13.4674,[215]13.4578,[216]13.4353,[217]13.4374,[218]13.3976,[219]13.4248,[220]13.3922,[221]13.3651,[222]13.3419,[223]13.3077,[224]13.2542,[225]13.2403,[226]13.2398,[227]13.2233,[228]13.2350,[229]13.1848,[230]13.1491,[231]13.1361,[232]13.1048,[233]13.0692,[234]13.0449,[235]13.0035,[236]12.9909,[237]12.9863,[238]12.9958,[239]12.9736,[240]12.9435,[241]12.9377,[242]12.8966,[243]12.8585,[244]12.8457,[245]12.8341,[246]12.8416,[247]12.8441,[248]12.8708,[249]12.8957,[250]12.9189,[251]12.9414,[252]12.9611,[253]12.9984,[254]13.0248,[255]13.0445,[256]13.0718,
|
| 13 |
+
4.12.508.195 I Final estimate: PPL = 13.0718 +/- 0.16258
|
| 14 |
+
|
results/ppl_std_hfconv.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
0.00.350.172 W load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
|
| 5 |
+
0.01.333.570 I cmn init: llama threadpool init, n_threads = 64
|
| 6 |
+
0.01.410.456 I
|
| 7 |
+
0.01.410.625 I system_info: n_threads = 64 (n_threads_batch = 64) / 80 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
| 8 |
+
0.01.410.641 I perplexity: tokenizing the input ..
|
| 9 |
+
0.02.026.275 I perplexity: tokenization took 615.627 ms
|
| 10 |
+
0.02.026.496 I perplexity: calculating perplexity over 64 chunks, n_ctx=512, batch_size=2048, n_seq=4
|
| 11 |
+
0.07.853.203 I perplexity: 5.83 seconds per pass - ETA 1.55 minutes
|
| 12 |
+
[1]8.1805,[2]12.6373,[3]11.8638,[4]12.3617,[5]12.6663,[6]13.1201,[7]13.8095,[8]14.2267,[9]14.9397,[10]15.5506,[11]15.7619,[12]16.0234,[13]16.0860,[14]15.7798,[15]15.4154,[16]15.8763,[17]14.5611,[18]14.6866,[19]14.8310,[20]14.5036,[21]14.3380,[22]14.1237,[23]13.6626,[24]13.2579,[25]12.8837,[26]12.7161,[27]12.0963,[28]11.9504,[29]11.9826,[30]11.7738,[31]11.9129,[32]11.9238,[33]12.0048,[34]12.2665,[35]12.3705,[36]12.5638,[37]12.6779,[38]12.8489,[39]12.7929,[40]12.7321,[41]12.7124,[42]12.6202,[43]12.7620,[44]12.7058,[45]12.6333,[46]12.8092,[47]12.7222,[48]12.9243,[49]12.9285,[50]12.9425,[51]13.0944,[52]13.1409,[53]13.1278,[54]13.1657,[55]13.2771,[56]13.3026,[57]13.3324,[58]13.3268,[59]13.4020,[60]13.4787,[61]13.5656,[62]13.6147,[63]13.6801,[64]13.8127,
|
| 13 |
+
1.21.268.982 I Final estimate: PPL = 13.8127 +/- 0.34818
|
| 14 |
+
|
results/ppl_std_official.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
0.00.331.757 W load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
|
| 5 |
+
0.01.307.157 I cmn init: llama threadpool init, n_threads = 64
|
| 6 |
+
0.01.376.421 I
|
| 7 |
+
0.01.376.592 I system_info: n_threads = 64 (n_threads_batch = 64) / 80 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
| 8 |
+
0.01.376.598 I perplexity: tokenizing the input ..
|
| 9 |
+
0.01.949.421 I perplexity: tokenization took 572.815 ms
|
| 10 |
+
0.01.949.645 I perplexity: calculating perplexity over 64 chunks, n_ctx=512, batch_size=2048, n_seq=4
|
| 11 |
+
0.07.523.678 I perplexity: 5.57 seconds per pass - ETA 1.48 minutes
|
| 12 |
+
[1]8.1805,[2]12.6373,[3]11.8638,[4]12.3617,[5]12.6663,[6]13.1201,[7]13.8095,[8]14.2267,[9]14.9397,[10]15.5506,[11]15.7619,[12]16.0234,[13]16.0860,[14]15.7798,[15]15.4154,[16]15.8763,[17]14.5611,[18]14.6866,[19]14.8310,[20]14.5036,[21]14.3380,[22]14.1237,[23]13.6626,[24]13.2579,[25]12.8837,[26]12.7161,[27]12.0963,[28]11.9504,[29]11.9826,[30]11.7738,[31]11.9129,[32]11.9238,[33]12.0048,[34]12.2665,[35]12.3705,[36]12.5638,[37]12.6779,[38]12.8489,[39]12.7929,[40]12.7321,[41]12.7124,[42]12.6202,[43]12.7620,[44]12.7058,[45]12.6333,[46]12.8092,[47]12.7222,[48]12.9243,[49]12.9285,[50]12.9425,[51]13.0944,[52]13.1409,[53]13.1278,[54]13.1657,[55]13.2771,[56]13.3026,[57]13.3324,[58]13.3268,[59]13.4020,[60]13.4787,[61]13.5656,[62]13.6147,[63]13.6801,[64]13.8127,
|
| 13 |
+
1.16.478.757 I Final estimate: PPL = 13.8127 +/- 0.34818
|
| 14 |
+
|
results/ppl_vq3.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
0.00.329.008 W load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect
|
| 5 |
+
0.02.197.753 I cmn init: llama threadpool init, n_threads = 64
|
| 6 |
+
0.02.342.303 I
|
| 7 |
+
0.02.342.454 I system_info: n_threads = 64 (n_threads_batch = 64) / 80 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | AMX_INT8 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 |
|
| 8 |
+
0.02.342.460 I perplexity: tokenizing the input ..
|
| 9 |
+
0.02.976.031 I perplexity: tokenization took 633.56 ms
|
| 10 |
+
0.02.976.262 I perplexity: calculating perplexity over 256 chunks, n_ctx=512, batch_size=2048, n_seq=4
|
| 11 |
+
0.10.267.377 I perplexity: 7.29 seconds per pass - ETA 7.77 minutes
|
| 12 |
+
[1]8.7929,[2]13.6217,[3]12.9068,[4]13.5700,[5]13.7100,[6]14.5695,[7]15.0196,[8]15.3338,[9]16.2651,[10]16.7569,[11]16.9390,[12]17.1636,[13]17.1726,[14]16.8770,[15]16.4804,[16]16.8638,[17]15.5087,[18]15.5547,[19]15.6173,[20]15.4209,[21]15.1014,[22]14.8532,[23]14.3827,[24]13.9584,[25]13.6135,[26]13.4733,[27]12.7868,[28]12.6315,[29]12.6185,[30]12.4117,[31]12.5078,[32]12.4712,[33]12.6295,[34]12.8252,[35]12.8881,[36]13.0594,[37]13.2544,[38]13.4175,[39]13.3331,[40]13.2822,[41]13.2789,[42]13.2037,[43]13.3499,[44]13.2897,[45]13.2097,[46]13.3727,[47]13.2571,[48]13.5041,[49]13.5273,[50]13.5432,[51]13.7143,[52]13.7416,[53]13.7842,[54]13.8445,[55]13.9728,[56]13.9910,[57]14.0223,[58]14.0275,[59]14.0978,[60]14.2088,[61]14.3146,[62]14.3945,[63]14.4258,[64]14.5472,[65]14.7084,[66]14.8671,[67]15.0551,[68]15.1368,[69]15.0460,[70]15.0963,[71]15.1601,[72]15.2360,[73]15.3417,[74]15.3028,[75]15.3598,[76]15.3021,[77]15.2352,[78]15.2288,[79]15.1920,[80]15.0102,[81]14.9372,[82]14.8559,[83]14.8115,[84]14.8658,[85]14.9223,[86]14.8710,[87]14.8965,[88]14.8732,[89]14.9157,[90]14.9081,[91]14.8268,[92]14.7959,[93]14.7643,[94]14.7099,[95]14.7260,[96]14.7267,[97]14.7503,[98]14.7582,[99]14.6889,[100]14.6781,[101]14.6457,[102]14.6859,[103]14.7290,[104]14.7364,[105]14.8379,[106]14.7710,[107]14.8002,[108]14.7649,[109]14.8008,[110]14.8906,[111]14.9657,[112]15.1198,[113]15.2687,[114]15.2905,[115]15.1885,[116]15.2599,[117]15.3055,[118]15.2849,[119]15.2515,[120]15.1969,[121]15.1514,[122]15.1257,[123]15.1242,[124]15.1094,[125]15.0663,[126]14.9979,[127]14.9375,[128]14.8934,[129]14.8131,[130]14.7801,[131]14.6824,[132]14.5994,[133]14.5697,[134]14.4928,[135]14.5071,[136]14.4719,[137]14.4675,[138]14.4815,[139]14.4229,[140]14.3543,[141]14.3266,[142]14.3265,[143]14.3652,[144]14.4080,[145]14.3519,[146]14.3288,[147]14.3598,[148]14.3257,[149]14.2864,[150]14.2510,[151]14.1727,[152]14.0737,[153]13.9540,[154]13.8810,[155]13.8515,[156]13.8188,[157]13.8065,[158]13.7935,[159]13.7131,[160]13.6627,[161]13.6123,[162]13.5890,[163]13.5303,[164]13.5117,[165]13.5173,[166]13.4944,[167]13.5238,[168]13.4776,[169]13.4690,[170]13.4389,[171]13.4906,[172]13.4780,[173]13.5122,[174]13.5233,[175]13.5881,[176]13.6488,[177]13.6945,[178]13.7774,[179]13.8304,[180]13.9246,[181]13.9791,[182]13.9873,[183]14.0049,[184]14.0304,[185]14.0753,[186]14.1076,[187]14.1439,[188]14.1792,[189]14.1911,[190]14.2350,[191]14.2454,[192]14.2484,[193]14.2771,[194]14.3101,[195]14.3538,[196]14.3359,[197]14.4105,[198]14.4444,[199]14.5064,[200]14.5157,[201]14.5251,[202]14.5004,[203]14.4779,[204]14.4621,[205]14.4203,[206]14.4651,[207]14.4840,[208]14.4739,[209]14.4329,[210]14.3890,[211]14.3604,[212]14.3039,[213]14.3318,[214]14.3252,[215]14.3046,[216]14.2726,[217]14.2709,[218]14.2220,[219]14.2528,[220]14.2227,[221]14.1915,[222]14.1632,[223]14.1279,[224]14.0748,[225]14.0558,[226]14.0571,[227]14.0363,[228]14.0452,[229]13.9968,[230]13.9570,[231]13.9471,[232]13.9116,[233]13.8724,[234]13.8420,[235]13.7917,[236]13.7735,[237]13.7696,[238]13.7836,[239]13.7603,[240]13.7166,[241]13.7108,[242]13.6643,[243]13.6268,[244]13.6154,[245]13.6002,[246]13.6052,[247]13.6121,[248]13.6366,[249]13.6810,[250]13.7069,[251]13.7330,[252]13.7495,[253]13.7844,[254]13.8035,[255]13.8204,[256]13.8465,
|
| 13 |
+
6.06.419.702 I Final estimate: PPL = 13.8465 +/- 0.17343
|
| 14 |
+
|
results/rss_base_q8dq6.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-base-Q8dQ6.gguf -p Hello -n 8 -t 64 --no-warmup -ngl 0 --temp 0"
|
| 5 |
+
User time (seconds): 39.65
|
| 6 |
+
System time (seconds): 6.42
|
| 7 |
+
Percent of CPU this job got: 1712%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.69
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 9230388
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 1
|
| 16 |
+
Minor (reclaiming a frame) page faults: 588949
|
| 17 |
+
Voluntary context switches: 1058
|
| 18 |
+
Involuntary context switches: 897
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 32
|
| 21 |
+
File system outputs: 8
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
results/rss_bf16.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-bf16.gguf -p Hello -n 8 -t 64 --no-warmup -ngl 0 --temp 0"
|
| 5 |
+
User time (seconds): 36.60
|
| 6 |
+
System time (seconds): 2.72
|
| 7 |
+
Percent of CPU this job got: 1106%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:03.55
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 16467308
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 0
|
| 16 |
+
Minor (reclaiming a frame) page faults: 509402
|
| 17 |
+
Voluntary context switches: 955
|
| 18 |
+
Involuntary context switches: 280
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 0
|
| 21 |
+
File system outputs: 8
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
results/rss_q4_k_m.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-Q4_K_M.gguf -p Hello -n 8 -t 64 --no-warmup -ngl 0 --temp 0"
|
| 5 |
+
User time (seconds): 132.44
|
| 6 |
+
System time (seconds): 7.61
|
| 7 |
+
Percent of CPU this job got: 1770%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:07.91
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 9162788
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 0
|
| 16 |
+
Minor (reclaiming a frame) page faults: 1421704
|
| 17 |
+
Voluntary context switches: 3631
|
| 18 |
+
Involuntary context switches: 4557
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 0
|
| 21 |
+
File system outputs: 8
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
results/rss_q8_0.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-Q8_0.gguf -p Hello -n 8 -t 64 --no-warmup -ngl 0 --temp 0"
|
| 5 |
+
User time (seconds): 126.09
|
| 6 |
+
System time (seconds): 8.92
|
| 7 |
+
Percent of CPU this job got: 3360%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:04.01
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 9770724
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 0
|
| 16 |
+
Minor (reclaiming a frame) page faults: 801535
|
| 17 |
+
Voluntary context switches: 1677
|
| 18 |
+
Involuntary context switches: 6386
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 0
|
| 21 |
+
File system outputs: 8
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
results/rss_vq3.txt
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
warning: no usable GPU found, --gpu-layers option will be ignored
|
| 2 |
+
warning: one possible reason is that llama.cpp was compiled without GPU support
|
| 3 |
+
warning: consult docs/build.md for compilation instructions
|
| 4 |
+
Command being timed: "./llama.cpp/build/bin/llama-cli -m models/Ling-3.0-tiny-VQ3.gguf -p Hello -n 8 -t 64 --no-warmup -ngl 0 --temp 0"
|
| 5 |
+
User time (seconds): 35.87
|
| 6 |
+
System time (seconds): 6.00
|
| 7 |
+
Percent of CPU this job got: 1760%
|
| 8 |
+
Elapsed (wall clock) time (h:mm:ss or m:ss): 0:02.37
|
| 9 |
+
Average shared text size (kbytes): 0
|
| 10 |
+
Average unshared data size (kbytes): 0
|
| 11 |
+
Average stack size (kbytes): 0
|
| 12 |
+
Average total size (kbytes): 0
|
| 13 |
+
Maximum resident set size (kbytes): 6120752
|
| 14 |
+
Average resident set size (kbytes): 0
|
| 15 |
+
Major (requiring I/O) page faults: 0
|
| 16 |
+
Minor (reclaiming a frame) page faults: 605491
|
| 17 |
+
Voluntary context switches: 1247
|
| 18 |
+
Involuntary context switches: 1133
|
| 19 |
+
Swaps: 0
|
| 20 |
+
File system inputs: 0
|
| 21 |
+
File system outputs: 8
|
| 22 |
+
Socket messages sent: 0
|
| 23 |
+
Socket messages received: 0
|
| 24 |
+
Signals delivered: 0
|
| 25 |
+
Page size (bytes): 4096
|
| 26 |
+
Exit status: 0
|
scripts/build_ling_vq3.py
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
build_ling_vq3.py — Build the inclusionAI Ling-3.0-tiny LUT-VQ3-64 hybrid GGUF.
|
| 4 |
+
|
| 5 |
+
Based on llama-modchip's scripts/build_nemotron_vq3.py, adapted for the
|
| 6 |
+
BailingMoeV3 (bailing_hybrid) architecture and for the VQ3 port to upstream
|
| 7 |
+
llama.cpp @ 61dbf492 (GGML_TYPE_VQ3_0 = 44 in the port, 24 B / 64 W, 3.00 bpw).
|
| 8 |
+
|
| 9 |
+
Recipes:
|
| 10 |
+
hybrid (default, domain-aware — mirrors the modchip VQ3-64 layout):
|
| 11 |
+
blk.*.ffn_gate_exps.weight + blk.*.ffn_up_exps.weight -> LUT-VQ3-64 (3.00 bpw)
|
| 12 |
+
blk.*.ffn_down_exps.weight -> Q6_K (6.56 bpw)
|
| 13 |
+
attention / shared experts / dense ffn / embeddings -> Q8_0 (8.50 bpw)
|
| 14 |
+
max (--down-vq3):
|
| 15 |
+
additionally blk.*.ffn_down_exps.weight -> LUT-VQ3-64 (3.00 bpw)
|
| 16 |
+
|
| 17 |
+
The routed-expert tensors are encoded directly from the uncompressed BF16 master
|
| 18 |
+
GGUF (inclusionAI/Ling-3.0-tiny-bf16.gguf) with the modchip's OpenMP AVX-512
|
| 19 |
+
encoder (vq3/vq3_enc_lib.c, 128-entry 3D Lloyd-Max codebook).
|
| 20 |
+
|
| 21 |
+
Usage:
|
| 22 |
+
python3 build_ling_vq3.py --base-gguf models/Ling-3.0-tiny-base.gguf \
|
| 23 |
+
--bf16-gguf models/Ling-3.0-tiny-bf16.gguf \
|
| 24 |
+
--output models/Ling-3.0-tiny-VQ3.gguf
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
import argparse
|
| 28 |
+
import ctypes
|
| 29 |
+
import mmap
|
| 30 |
+
import os
|
| 31 |
+
import struct
|
| 32 |
+
import subprocess
|
| 33 |
+
import sys
|
| 34 |
+
import time
|
| 35 |
+
from pathlib import Path
|
| 36 |
+
|
| 37 |
+
import numpy as np
|
| 38 |
+
|
| 39 |
+
GGML_TYPE_BF16 = 30
|
| 40 |
+
GGML_TYPE_VQ3_0 = 44 # llama-modchip port: appended after upstream GGML_TYPE_F8_E4M3 = 43
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def load_vq3_encoder(lib_path: str = "") -> ctypes.CDLL:
|
| 44 |
+
here = Path(__file__).resolve().parent
|
| 45 |
+
# encoder source lives next to this script (HF repo layout) or in a llama-modchip checkout
|
| 46 |
+
so_candidates = [
|
| 47 |
+
Path(lib_path) if lib_path else None,
|
| 48 |
+
here.parent / "vq3" / "libvq3_enc.so",
|
| 49 |
+
here.parent.parent / "llama-modchip" / "vq3" / "libvq3_enc.so",
|
| 50 |
+
]
|
| 51 |
+
so_path = next((c for c in so_candidates if c and c.is_file()), None)
|
| 52 |
+
src_candidates = [
|
| 53 |
+
here.parent / "vq3" / "vq3_enc_lib.c",
|
| 54 |
+
here.parent.parent / "llama-modchip" / "vq3" / "vq3_enc_lib.c",
|
| 55 |
+
]
|
| 56 |
+
c_src = next((c for c in src_candidates if c.is_file()), None)
|
| 57 |
+
if so_path is None:
|
| 58 |
+
if not c_src:
|
| 59 |
+
raise FileNotFoundError(f"Missing encoder library and source {src_candidates}")
|
| 60 |
+
so_path = c_src.with_suffix(".so").parent / "libvq3_enc.so"
|
| 61 |
+
print(f"[*] Compiling {c_src} -> {so_path} ...", flush=True)
|
| 62 |
+
subprocess.check_call([
|
| 63 |
+
"gcc", "-O3", "-march=native", "-mavx512f", "-mf16c",
|
| 64 |
+
"-fopenmp", "-shared", "-fPIC", str(c_src), "-lm", "-o", str(so_path),
|
| 65 |
+
])
|
| 66 |
+
lib = ctypes.CDLL(str(so_path))
|
| 67 |
+
lib.quantize_bf16_to_vq3_0_omp.argtypes = [
|
| 68 |
+
ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_int64,
|
| 69 |
+
]
|
| 70 |
+
lib.quantize_bf16_to_vq3_0_omp.restype = None
|
| 71 |
+
return lib
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def parse_gguf(f):
|
| 75 |
+
"""Minimal GGUF v2/v3 reader: returns metadata KV blob + tensor directory."""
|
| 76 |
+
f.seek(0)
|
| 77 |
+
magic, version, n_tensors, n_kv = struct.unpack("<4sIQQ", f.read(24))
|
| 78 |
+
if magic != b"GGUF":
|
| 79 |
+
raise ValueError(f"Not a GGUF file (magic={magic!r})")
|
| 80 |
+
|
| 81 |
+
def read_str():
|
| 82 |
+
length = struct.unpack("<Q", f.read(8))[0]
|
| 83 |
+
return f.read(length).decode("utf-8", errors="replace")
|
| 84 |
+
|
| 85 |
+
def skip_val(vtype):
|
| 86 |
+
if vtype in (0, 1, 7):
|
| 87 |
+
f.seek(1, 1)
|
| 88 |
+
elif vtype in (2, 3):
|
| 89 |
+
f.seek(2, 1)
|
| 90 |
+
elif vtype in (4, 5, 6):
|
| 91 |
+
f.seek(4, 1)
|
| 92 |
+
elif vtype in (10, 11, 12):
|
| 93 |
+
f.seek(8, 1)
|
| 94 |
+
elif vtype == 8:
|
| 95 |
+
length = struct.unpack("<Q", f.read(8))[0]
|
| 96 |
+
f.seek(length, 1)
|
| 97 |
+
elif vtype == 9:
|
| 98 |
+
atype, alen = struct.unpack("<IQ", f.read(12))
|
| 99 |
+
for _ in range(alen):
|
| 100 |
+
skip_val(atype)
|
| 101 |
+
else:
|
| 102 |
+
raise ValueError(f"Unknown GGUF metadata type {vtype}")
|
| 103 |
+
|
| 104 |
+
for _ in range(n_kv):
|
| 105 |
+
_ = read_str()
|
| 106 |
+
vtype = struct.unpack("<I", f.read(4))[0]
|
| 107 |
+
skip_val(vtype)
|
| 108 |
+
|
| 109 |
+
kv_end = f.tell()
|
| 110 |
+
tensors = []
|
| 111 |
+
for _ in range(n_tensors):
|
| 112 |
+
name = read_str()
|
| 113 |
+
n_dims = struct.unpack("<I", f.read(4))[0]
|
| 114 |
+
dims = [struct.unpack("<Q", f.read(8))[0] for _ in range(n_dims)]
|
| 115 |
+
ttype, offset = struct.unpack("<IQ", f.read(12))
|
| 116 |
+
tensors.append({
|
| 117 |
+
"name": name,
|
| 118 |
+
"n_dims": n_dims,
|
| 119 |
+
"dims": dims,
|
| 120 |
+
"type": ttype,
|
| 121 |
+
"offset": offset,
|
| 122 |
+
})
|
| 123 |
+
header_end = f.tell()
|
| 124 |
+
alignment = 32
|
| 125 |
+
data_start = (header_end + alignment - 1) // alignment * alignment
|
| 126 |
+
return version, n_tensors, n_kv, kv_end, tensors, header_end, data_start
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def main() -> int:
|
| 130 |
+
ap = argparse.ArgumentParser(description="Build Ling-3.0-tiny LUT-VQ3-64 hybrid GGUF")
|
| 131 |
+
ap.add_argument("--base-gguf", required=True,
|
| 132 |
+
help="Quantized base GGUF (e.g. Q8_0 with --tensor-type ffn_down_exps=Q6_K)")
|
| 133 |
+
ap.add_argument("--bf16-gguf", required=True,
|
| 134 |
+
help="Uncompressed BF16 master GGUF (VQ3 encode source)")
|
| 135 |
+
ap.add_argument("--output", required=True, help="Output VQ3 hybrid GGUF path")
|
| 136 |
+
ap.add_argument("--down-vq3", action="store_true",
|
| 137 |
+
help="Also encode ffn_down_exps to VQ3-64 (aggressive 'max' recipe)")
|
| 138 |
+
ap.add_argument("--lib", default=os.environ.get("VQ3_ENC_LIB", ""),
|
| 139 |
+
help="Optional path to libvq3_enc.so")
|
| 140 |
+
args = ap.parse_args()
|
| 141 |
+
|
| 142 |
+
lib = load_vq3_encoder(args.lib)
|
| 143 |
+
|
| 144 |
+
vq3_match = ("ffn_gate_exps", "ffn_up_exps") + (("ffn_down_exps",) if args.down_vq3 else ())
|
| 145 |
+
|
| 146 |
+
t0 = time.time()
|
| 147 |
+
with open(args.base_gguf, "rb") as ff, open(args.bf16_gguf, "rb") as fb:
|
| 148 |
+
_, _, _, kv_end, f_tensors, f_hdr_end, f_data_start = parse_gguf(ff)
|
| 149 |
+
_, _, _, _, b_tensors, _, b_data_start = parse_gguf(fb)
|
| 150 |
+
bf16_map = {t["name"]: t for t in b_tensors}
|
| 151 |
+
|
| 152 |
+
sorted_ft = sorted(f_tensors, key=lambda x: x["offset"])
|
| 153 |
+
ff.seek(0, os.SEEK_END)
|
| 154 |
+
f_file_size = ff.tell()
|
| 155 |
+
|
| 156 |
+
new_offset = 0
|
| 157 |
+
alignment = 32
|
| 158 |
+
vq3_target_total = 0
|
| 159 |
+
for t in f_tensors:
|
| 160 |
+
n_el = 1
|
| 161 |
+
for d in t["dims"]:
|
| 162 |
+
n_el *= d
|
| 163 |
+
t["n_el"] = n_el
|
| 164 |
+
if any(m in t["name"] for m in vq3_match) and t["name"].endswith("exps.weight"):
|
| 165 |
+
if n_el % 64:
|
| 166 |
+
raise ValueError(f"{t['name']}: {n_el} not divisible by 64")
|
| 167 |
+
t["new_type"] = GGML_TYPE_VQ3_0
|
| 168 |
+
t["new_bytes"] = (n_el // 64) * 24
|
| 169 |
+
t["old_bytes"] = (n_el // 64) * 30 if t["type"] == GGML_TYPE_BF16 else None
|
| 170 |
+
if t["old_bytes"] is None:
|
| 171 |
+
t["old_bytes"] = next(
|
| 172 |
+
(sorted_ft[i + 1]["offset"] - t["offset"] for i in range(len(sorted_ft) - 1)
|
| 173 |
+
if sorted_ft[i] is t), f_file_size - (f_data_start + t["offset"]))
|
| 174 |
+
vq3_target_total += 1
|
| 175 |
+
else:
|
| 176 |
+
t["new_type"] = t["type"]
|
| 177 |
+
idx_s = sorted_ft.index(t)
|
| 178 |
+
if idx_s + 1 < len(sorted_ft):
|
| 179 |
+
span = sorted_ft[idx_s + 1]["offset"] - t["offset"]
|
| 180 |
+
else:
|
| 181 |
+
span = f_file_size - (f_data_start + t["offset"])
|
| 182 |
+
t["old_bytes"] = span
|
| 183 |
+
t["new_bytes"] = span
|
| 184 |
+
t["new_offset"] = new_offset
|
| 185 |
+
padded = (t["new_bytes"] + alignment - 1) // alignment * alignment
|
| 186 |
+
new_offset += padded
|
| 187 |
+
|
| 188 |
+
out_gb = (f_data_start + new_offset) / (1024 ** 3)
|
| 189 |
+
print(f"Parsed headers: {len(f_tensors)} tensors, encoding {vq3_target_total} expert tensors "
|
| 190 |
+
f"to VQ3_0(44), output size ~{out_gb:.2f} GiB", flush=True)
|
| 191 |
+
|
| 192 |
+
mm_f = mmap.mmap(ff.fileno(), 0, access=mmap.ACCESS_READ)
|
| 193 |
+
mm_b = mmap.mmap(fb.fileno(), 0, access=mmap.ACCESS_READ)
|
| 194 |
+
|
| 195 |
+
with open(args.output, "wb") as fout:
|
| 196 |
+
# metadata KV (verbatim from base) + rewritten tensor directory
|
| 197 |
+
ff.seek(0)
|
| 198 |
+
fout.write(ff.read(kv_end))
|
| 199 |
+
for t in f_tensors:
|
| 200 |
+
nb = t["name"].encode("utf-8")
|
| 201 |
+
fout.write(struct.pack("<Q", len(nb)))
|
| 202 |
+
fout.write(nb)
|
| 203 |
+
fout.write(struct.pack("<I", t["n_dims"]))
|
| 204 |
+
for d in t["dims"]:
|
| 205 |
+
fout.write(struct.pack("<Q", d))
|
| 206 |
+
fout.write(struct.pack("<IQ", t["new_type"], t["new_offset"]))
|
| 207 |
+
assert fout.tell() == f_hdr_end, (fout.tell(), f_hdr_end)
|
| 208 |
+
if f_data_start > f_hdr_end:
|
| 209 |
+
fout.write(b"\x00" * (f_data_start - f_hdr_end))
|
| 210 |
+
|
| 211 |
+
vq3_cnt = 0
|
| 212 |
+
for t in f_tensors:
|
| 213 |
+
target_pos = f_data_start + t["new_offset"]
|
| 214 |
+
cur_pos = fout.tell()
|
| 215 |
+
if target_pos > cur_pos:
|
| 216 |
+
fout.write(b"\x00" * (target_pos - cur_pos))
|
| 217 |
+
assert fout.tell() == target_pos
|
| 218 |
+
|
| 219 |
+
if t["new_type"] == GGML_TYPE_VQ3_0:
|
| 220 |
+
bt = bf16_map[t["name"]]
|
| 221 |
+
t_type = bt["type"]
|
| 222 |
+
assert t_type == GGML_TYPE_BF16, \
|
| 223 |
+
f"Expected BF16 ({GGML_TYPE_BF16}) for {t['name']}, got {t_type}"
|
| 224 |
+
n_el = t["n_el"]
|
| 225 |
+
ncol = t["dims"][0]
|
| 226 |
+
nrow = n_el // ncol
|
| 227 |
+
b_off = b_data_start + bt["offset"]
|
| 228 |
+
src_ptr = np.frombuffer(mm_b, dtype=np.uint8, count=n_el * 2,
|
| 229 |
+
offset=b_off).ctypes.data
|
| 230 |
+
out_buf = np.empty(t["new_bytes"], dtype=np.uint8)
|
| 231 |
+
dst_ptr = out_buf.ctypes.data
|
| 232 |
+
ts = time.time()
|
| 233 |
+
lib.quantize_bf16_to_vq3_0_omp(src_ptr, dst_ptr, nrow, ncol)
|
| 234 |
+
fout.write(out_buf.tobytes())
|
| 235 |
+
vq3_cnt += 1
|
| 236 |
+
print(f"[{vq3_cnt:2d}/{vq3_target_total}] {t['name']} "
|
| 237 |
+
f"({n_el / 1e6:.1f}M w) BF16 -> VQ3_0 in {time.time() - ts:.2f}s",
|
| 238 |
+
flush=True)
|
| 239 |
+
else:
|
| 240 |
+
f_off = f_data_start + t["offset"]
|
| 241 |
+
fout.write(mm_f[f_off:f_off + t["old_bytes"]])
|
| 242 |
+
|
| 243 |
+
rem = fout.tell() % alignment
|
| 244 |
+
if rem:
|
| 245 |
+
fout.write(b"\x00" * (alignment - rem))
|
| 246 |
+
|
| 247 |
+
mm_f.close()
|
| 248 |
+
mm_b.close()
|
| 249 |
+
|
| 250 |
+
print(f"DONE in {time.time() - t0:.1f}s: {args.output} "
|
| 251 |
+
f"({os.path.getsize(args.output) / (1024 ** 3):.2f} GiB)", flush=True)
|
| 252 |
+
return 0
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
if __name__ == "__main__":
|
| 256 |
+
raise SystemExit(main())
|
scripts/run_experiments.sh
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# One-shot experiment suite for Ling-3.0-tiny VQ3 vs standard quantizations
|
| 3 |
+
# Usage: run_experiments.sh (from work root)
|
| 4 |
+
set -u
|
| 5 |
+
cd "$(dirname "$0")/../.."
|
| 6 |
+
BIN=./llama.cpp/build/bin
|
| 7 |
+
CORP=corpora/wikitext-2-raw/wiki.test.raw
|
| 8 |
+
R=ling-vq3/results
|
| 9 |
+
mkdir -p "$R"
|
| 10 |
+
T=64
|
| 11 |
+
CHUNKS_PPL=256
|
| 12 |
+
CHUNKS_KLD=128
|
| 13 |
+
|
| 14 |
+
declare -A MODELS=(
|
| 15 |
+
[bf16]=models/Ling-3.0-tiny-bf16.gguf
|
| 16 |
+
[q8_0]=models/Ling-3.0-tiny-Q8_0.gguf
|
| 17 |
+
[q4_k_m]=models/Ling-3.0-tiny-Q4_K_M.gguf
|
| 18 |
+
[base_q8dq6]=models/Ling-3.0-tiny-base-Q8dQ6.gguf
|
| 19 |
+
[vq3]=models/Ling-3.0-tiny-VQ3.gguf
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
echo "=== [1/5] PPL (wikitext-2 test, $CHUNKS_PPL chunks) ==="
|
| 23 |
+
for k in bf16 q8_0 q4_k_m base_q8dq6 vq3; do
|
| 24 |
+
m=${MODELS[$k]}
|
| 25 |
+
echo "--- PPL $k ---"
|
| 26 |
+
$BIN/llama-perplexity -m "$m" -f "$CORP" -c 512 --chunks $CHUNKS_PPL -t $T -ngl 0 \
|
| 27 |
+
> "$R/ppl_$k.log" 2>&1
|
| 28 |
+
grep -E "Final estimate|error" "$R/ppl_$k.log" | tail -2
|
| 29 |
+
done
|
| 30 |
+
|
| 31 |
+
echo "=== [2/5] Baseline logits from BF16 (KLD reference, $CHUNKS_KLD chunks) ==="
|
| 32 |
+
$BIN/llama-perplexity -m models/Ling-3.0-tiny-bf16.gguf -f "$CORP" -c 512 --chunks $CHUNKS_KLD \
|
| 33 |
+
-t $T -ngl 0 --save-all-logits "$R/logits_bf16.bin" > "$R/logits_bf16.log" 2>&1
|
| 34 |
+
grep -E "Final estimate|error" "$R/logits_bf16.log" | tail -2
|
| 35 |
+
|
| 36 |
+
echo "=== [3/5] KLD vs BF16 ==="
|
| 37 |
+
for k in q8_0 q4_k_m base_q8dq6 vq3; do
|
| 38 |
+
m=${MODELS[$k]}
|
| 39 |
+
echo "--- KLD $k ---"
|
| 40 |
+
$BIN/llama-perplexity -m "$m" -f "$CORP" -c 512 --chunks $CHUNKS_KLD -t $T -ngl 0 \
|
| 41 |
+
--kl-divergence --kl-divergence-base "$R/logits_bf16.bin" > "$R/kld_$k.log" 2>&1
|
| 42 |
+
grep -E "Final estimate|error" "$R/kld_$k.log" | tail -2
|
| 43 |
+
done
|
| 44 |
+
|
| 45 |
+
echo "=== [4/5] llama-bench pp512/tg128 ==="
|
| 46 |
+
$BIN/llama-bench -t $T -p 512 -n 128 -r 2 -mlo 0 \
|
| 47 |
+
-m models/Ling-3.0-tiny-bf16.gguf \
|
| 48 |
+
-m models/Ling-3.0-tiny-Q8_0.gguf \
|
| 49 |
+
-m models/Ling-3.0-tiny-Q4_K_M.gguf \
|
| 50 |
+
-m models/Ling-3.0-tiny-base-Q8dQ6.gguf \
|
| 51 |
+
-m models/Ling-3.0-tiny-VQ3.gguf > "$R/bench.tsv" 2> "$R/bench.err"
|
| 52 |
+
cat "$R/bench.tsv"
|
| 53 |
+
|
| 54 |
+
echo "=== [5/5] Peak RSS via short generation ==="
|
| 55 |
+
for k in bf16 q8_0 q4_k_m base_q8dq6 vq3; do
|
| 56 |
+
m=${MODELS[$k]}
|
| 57 |
+
/usr/bin/time -v $BIN/llama-cli -m "$m" -p "Hello" -n 8 -t $T --no-warmup -ngl 0 --temp 0 \
|
| 58 |
+
> /dev/null 2> "$R/rss_$k.txt"
|
| 59 |
+
rss=$(grep "Maximum resident" "$R/rss_$k.txt" | awk '{print $6}')
|
| 60 |
+
echo "peak RSS $k: $rss KB"
|
| 61 |
+
done
|
| 62 |
+
|
| 63 |
+
echo "=== ALL DONE ==="
|
tests/test_vq3_port.c
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/*
|
| 2 |
+
* test_vq3_port.c — Verification test for the llama-modchip LUT-VQ3-64 port
|
| 3 |
+
* (GGML_TYPE_VQ3_0 = 44 in this port) against the built ggml static libs.
|
| 4 |
+
*
|
| 5 |
+
* 1. type traits sanity (vq3_0 / 64 / 24 B / quantized)
|
| 6 |
+
* 2. ggml_quantize_chunk(VQ3_0) round-trip SQNR / cosine on heavy-tailed data
|
| 7 |
+
* 3. vec_dot(VQ3_0 x Q8_0) vs F32 reference (needs ggml_init() for the FP16 table!)
|
| 8 |
+
* 4. scalar-fallback vs AVX-512 VBMI kernel byte-exact quantizer equivalence
|
| 9 |
+
*
|
| 10 |
+
* Build:
|
| 11 |
+
* gcc -O2 -march=native -mf16c tests/test_vq3_port.c -I ../llama.cpp/ggml/include \
|
| 12 |
+
* -o test_vq3_port ../llama.cpp/build/ggml/src/libggml*.a -lm -lstdc++ -lgomp -lpthread
|
| 13 |
+
*/
|
| 14 |
+
#include <stdio.h>
|
| 15 |
+
#include <stdlib.h>
|
| 16 |
+
#include <string.h>
|
| 17 |
+
#include <math.h>
|
| 18 |
+
#include <stdint.h>
|
| 19 |
+
#include "ggml.h"
|
| 20 |
+
#include "ggml-cpu.h"
|
| 21 |
+
|
| 22 |
+
static uint64_t rng_state = 0x9E3779B97F4A7C15ULL;
|
| 23 |
+
static double frand(void) {
|
| 24 |
+
rng_state ^= rng_state << 13; rng_state ^= rng_state >> 7; rng_state ^= rng_state << 17;
|
| 25 |
+
return (double)(rng_state >> 11) / 9007199254740992.0;
|
| 26 |
+
}
|
| 27 |
+
static double gauss(void) {
|
| 28 |
+
double u1 = frand(), u2 = frand();
|
| 29 |
+
if (u1 < 1e-300) u1 = 1e-300;
|
| 30 |
+
return sqrt(-2.0 * log(u1)) * cos(2.0 * M_PI * u2);
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
int main(void) {
|
| 34 |
+
static char init_pool[1 << 20];
|
| 35 |
+
struct ggml_init_params ip = { sizeof(init_pool), init_pool, true };
|
| 36 |
+
struct ggml_context * ctx = ggml_init(ip);
|
| 37 |
+
ggml_cpu_init(); /* REQUIRED: fills ggml_table_f32_f16 used by GGML_CPU_FP16_TO_FP32 (upstream forces LUT, issue #30342) */
|
| 38 |
+
|
| 39 |
+
const enum ggml_type vt = GGML_TYPE_VQ3_0;
|
| 40 |
+
const struct ggml_type_traits * tt = ggml_get_type_traits(vt);
|
| 41 |
+
printf("type_name=%s blck_size=%d type_size=%zu is_quantized=%d\n",
|
| 42 |
+
tt->type_name, (int)tt->blck_size, tt->type_size, (int)tt->is_quantized);
|
| 43 |
+
if (strcmp(tt->type_name, "vq3_0") || tt->blck_size != 64 || tt->type_size != 24 || !tt->is_quantized) {
|
| 44 |
+
fprintf(stderr, "FAIL: type traits mismatch\n"); return 1;
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
const struct ggml_type_traits_cpu * tc = ggml_get_type_traits_cpu(vt);
|
| 48 |
+
if (!tc->from_float || !tc->vec_dot) { fprintf(stderr, "FAIL: cpu traits missing\n"); return 1; }
|
| 49 |
+
|
| 50 |
+
/* --- round trip ------------------------------------------------------- */
|
| 51 |
+
const int64_t n = 64 * 4096;
|
| 52 |
+
float * x = malloc(n * sizeof(float));
|
| 53 |
+
for (int64_t i = 0; i < n; ++i) {
|
| 54 |
+
double g = gauss() * 0.05;
|
| 55 |
+
if (frand() < 0.02) g *= 4.0;
|
| 56 |
+
x[i] = (float)g;
|
| 57 |
+
}
|
| 58 |
+
size_t nbytes = (n / tt->blck_size) * tt->type_size;
|
| 59 |
+
void * q = malloc(nbytes);
|
| 60 |
+
tc->from_float(x, q, n);
|
| 61 |
+
|
| 62 |
+
float * xd = malloc(n * sizeof(float));
|
| 63 |
+
tt->to_float(q, xd, n);
|
| 64 |
+
double sig = 0, err = 0, dxy = 0, dx2 = 0, dy2 = 0;
|
| 65 |
+
for (int64_t i = 0; i < n; ++i) {
|
| 66 |
+
double e = (double)x[i] - xd[i];
|
| 67 |
+
sig += (double)x[i] * x[i]; err += e * e;
|
| 68 |
+
dxy += (double)x[i] * xd[i]; dx2 += (double)x[i] * x[i]; dy2 += (double)xd[i] * xd[i];
|
| 69 |
+
}
|
| 70 |
+
double sqnr = 10.0 * log10(sig / err), cosim = dxy / (sqrt(dx2) * sqrt(dy2));
|
| 71 |
+
printf("round-trip: SQNR=%.2f dB cos=%.5f (%.3f bpw)\n", sqnr, cosim, 8.0 * (double)nbytes / n);
|
| 72 |
+
int fail = (sqnr < 11.0 || cosim < 0.95);
|
| 73 |
+
|
| 74 |
+
/* --- vec dot ---------------------------------------------------------- */
|
| 75 |
+
const int64_t nd = 64 * 16384;
|
| 76 |
+
float * a = malloc(nd * sizeof(float));
|
| 77 |
+
float * b = malloc(nd * sizeof(float));
|
| 78 |
+
for (int64_t i = 0; i < nd; ++i) {
|
| 79 |
+
a[i] = (float)(gauss() * 0.05 * ((frand() < 0.02) ? 4.0 : 1.0));
|
| 80 |
+
b[i] = (float)(gauss() * 0.05 * ((frand() < 0.02) ? 4.0 : 1.0));
|
| 81 |
+
}
|
| 82 |
+
void * qa = malloc(nd / 64 * 24);
|
| 83 |
+
void * qb8 = malloc(nd / 32 * 34);
|
| 84 |
+
tc->from_float(a, qa, nd);
|
| 85 |
+
ggml_get_type_traits_cpu(GGML_TYPE_Q8_0)->from_float(b, qb8, nd);
|
| 86 |
+
|
| 87 |
+
float dot_q = 0, dot_ref = 0, dot_dq = 0;
|
| 88 |
+
tc->vec_dot(nd, &dot_q, 0, qa, 0, qb8, 0, 1);
|
| 89 |
+
for (int64_t i = 0; i < nd; ++i) dot_ref += a[i] * b[i];
|
| 90 |
+
float * ad = malloc(nd * sizeof(float));
|
| 91 |
+
tt->to_float(qa, ad, nd);
|
| 92 |
+
for (int64_t i = 0; i < nd; ++i) dot_dq += ad[i] * b[i];
|
| 93 |
+
double rel = fabs((double)dot_q - dot_dq) / (fabs(dot_dq) + 1e-6);
|
| 94 |
+
/* noise scale: sqrt(sum a^2 b^2) — the f32 ref dot of independent samples is a
|
| 95 |
+
near-cancellation random walk, so normalize by the per-element noise scale */
|
| 96 |
+
double noise = 0; for (int64_t i = 0; i < nd; ++i) noise += (double)a[i] * a[i] * b[i] * b[i];
|
| 97 |
+
noise = sqrt(noise);
|
| 98 |
+
double rel2 = fabs((double)dot_dq - dot_ref) / noise;
|
| 99 |
+
printf("vec_dot: vq3xq8=%.10g dequant-f32=%.10g f32ref=%.10g |q8diff|/|dq|=%.5f |dequant-ref|/noise=%.4f\n",
|
| 100 |
+
dot_q, dot_dq, dot_ref, rel, rel2);
|
| 101 |
+
if (rel > 0.03) { fprintf(stderr, "FAIL: vec_dot deviates from dequant path beyond Q8 noise\n"); fail = 1; }
|
| 102 |
+
if (rel2 > 0.30) { fprintf(stderr, "FAIL: dequant dot error vs f32 too large\n"); fail = 1; }
|
| 103 |
+
|
| 104 |
+
if (!fail) printf("ALL OK\n"); else fprintf(stderr, "TEST FAILED\n");
|
| 105 |
+
ggml_free(ctx);
|
| 106 |
+
free(x); free(q); free(xd); free(a); free(b); free(qa); free(qb8); free(ad);
|
| 107 |
+
return fail;
|
| 108 |
+
}
|
vq3/vq3_codebook.h
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#ifndef GGML_VQ3_CODEBOOK_H
|
| 2 |
+
#define GGML_VQ3_CODEBOOK_H
|
| 3 |
+
|
| 4 |
+
#include <stdint.h>
|
| 5 |
+
|
| 6 |
+
#define GGML_VQ3_G (8.6373016238e-03f)
|
| 7 |
+
|
| 8 |
+
#if defined(_MSC_VER)
|
| 9 |
+
#define GGML_VQ3_ALIGN64 __declspec(align(64))
|
| 10 |
+
#else
|
| 11 |
+
#define GGML_VQ3_ALIGN64 __attribute__((aligned(64)))
|
| 12 |
+
#endif
|
| 13 |
+
|
| 14 |
+
static const int8_t ggml_vq3_cb[128][3] = {
|
| 15 |
+
{ -24, -19, -24 },
|
| 16 |
+
{ -37, -20, -123 },
|
| 17 |
+
{ -41, -18, 18 },
|
| 18 |
+
{ 23, -6, 27 },
|
| 19 |
+
{ 29, -15, -65 },
|
| 20 |
+
{ -3, 4, 0 },
|
| 21 |
+
{ -59, -25, 72 },
|
| 22 |
+
{ -68, -13, -66 },
|
| 23 |
+
{ 11, -54, -122 },
|
| 24 |
+
{ -84, -5, 30 },
|
| 25 |
+
{ 35, -51, 51 },
|
| 26 |
+
{ -19, 8, 50 },
|
| 27 |
+
{ 54, 9, 62 },
|
| 28 |
+
{ 12, -74, -55 },
|
| 29 |
+
{ -41, -73, -34 },
|
| 30 |
+
{ 16, 46, 19 },
|
| 31 |
+
{ -62, -70, -111 },
|
| 32 |
+
{ 69, -14, 116 },
|
| 33 |
+
{ -65, -73, 20 },
|
| 34 |
+
{ 70, 11, 12 },
|
| 35 |
+
{ 75, -2, -40 },
|
| 36 |
+
{ 55, 43, -65 },
|
| 37 |
+
{ -6, 62, -121 },
|
| 38 |
+
{ -47, 32, 16 },
|
| 39 |
+
{ 73, -64, 65 },
|
| 40 |
+
{ -22, 67, 121 },
|
| 41 |
+
{ 62, 61, -9 },
|
| 42 |
+
{ 5, 80, -18 },
|
| 43 |
+
{ -123, -46, 55 },
|
| 44 |
+
{ -119, -58, -56 },
|
| 45 |
+
{ -24, -120, -122 },
|
| 46 |
+
{ -57, 48, -72 },
|
| 47 |
+
{ -125, -52, -3 },
|
| 48 |
+
{ -30, 77, 27 },
|
| 49 |
+
{ -19, -126, -11 },
|
| 50 |
+
{ 43, -118, -118 },
|
| 51 |
+
{ -56, 69, -22 },
|
| 52 |
+
{ 52, -122, 119 },
|
| 53 |
+
{ 120, 6, -116 },
|
| 54 |
+
{ 123, -22, -59 },
|
| 55 |
+
{ 124, -26, 115 },
|
| 56 |
+
{ 127, 4, -10 },
|
| 57 |
+
{ -97, -121, -34 },
|
| 58 |
+
{ 119, 46, -48 },
|
| 59 |
+
{ 117, 54, -115 },
|
| 60 |
+
{ 7, 117, 76 },
|
| 61 |
+
{ 9, 120, 26 },
|
| 62 |
+
{ -122, -113, 102 },
|
| 63 |
+
{ -123, -114, 19 },
|
| 64 |
+
{ 5, 124, -58 },
|
| 65 |
+
{ 51, 122, -8 },
|
| 66 |
+
{ 100, -123, 51 },
|
| 67 |
+
{ 114, -111, -62 },
|
| 68 |
+
{ 62, 123, 52 },
|
| 69 |
+
{ -56, 119, -59 },
|
| 70 |
+
{ -122, 6, -31 },
|
| 71 |
+
{ -123, 8, 120 },
|
| 72 |
+
{ -121, 40, -121 },
|
| 73 |
+
{ 122, -120, -3 },
|
| 74 |
+
{ -127, 1, 15 },
|
| 75 |
+
{ 114, 114, 111 },
|
| 76 |
+
{ 121, 114, 28 },
|
| 77 |
+
{ -114, 97, 118 },
|
| 78 |
+
{ -119, 116, 48 },
|
| 79 |
+
{ 5, -19, 75 },
|
| 80 |
+
{ 16, -14, 126 },
|
| 81 |
+
{ 10, 8, -122 },
|
| 82 |
+
{ -18, -41, -70 },
|
| 83 |
+
{ 18, -35, -16 },
|
| 84 |
+
{ -13, -51, 18 },
|
| 85 |
+
{ -37, -47, 123 },
|
| 86 |
+
{ 29, 16, -25 },
|
| 87 |
+
{ -15, 8, -63 },
|
| 88 |
+
{ -15, 14, 120 },
|
| 89 |
+
{ -19, -71, 69 },
|
| 90 |
+
{ -75, -42, -18 },
|
| 91 |
+
{ 31, -64, 121 },
|
| 92 |
+
{ 60, -2, -118 },
|
| 93 |
+
{ -15, 37, -26 },
|
| 94 |
+
{ 1, 49, 66 },
|
| 95 |
+
{ 20, -81, 0 },
|
| 96 |
+
{ 43, 42, 120 },
|
| 97 |
+
{ 66, -41, 8 },
|
| 98 |
+
{ 63, -53, -98 },
|
| 99 |
+
{ 4, 58, -67 },
|
| 100 |
+
{ 63, -62, -38 },
|
| 101 |
+
{ -115, -30, -124 },
|
| 102 |
+
{ -108, -47, 118 },
|
| 103 |
+
{ 53, 59, -124 },
|
| 104 |
+
{ -54, 31, -125 },
|
| 105 |
+
{ 58, 65, 56 },
|
| 106 |
+
{ -62, 19, 123 },
|
| 107 |
+
{ -5, -119, 115 },
|
| 108 |
+
{ -66, 13, -23 },
|
| 109 |
+
{ -24, -121, 38 },
|
| 110 |
+
{ -31, -120, -66 },
|
| 111 |
+
{ -61, -113, 118 },
|
| 112 |
+
{ -63, 46, 65 },
|
| 113 |
+
{ 30, -121, 48 },
|
| 114 |
+
{ 35, -124, -54 },
|
| 115 |
+
{ 118, -18, 34 },
|
| 116 |
+
{ -76, -122, 46 },
|
| 117 |
+
{ 56, -122, -4 },
|
| 118 |
+
{ 118, 26, 62 },
|
| 119 |
+
{ 122, -48, -117 },
|
| 120 |
+
{ 115, 44, 120 },
|
| 121 |
+
{ 122, -56, -12 },
|
| 122 |
+
{ 38, 112, 124 },
|
| 123 |
+
{ -113, -115, -109 },
|
| 124 |
+
{ 124, -70, 44 },
|
| 125 |
+
{ 121, 55, 14 },
|
| 126 |
+
{ 63, 110, -59 },
|
| 127 |
+
{ 25, 123, -120 },
|
| 128 |
+
{ -47, 113, -121 },
|
| 129 |
+
{ -24, 124, -9 },
|
| 130 |
+
{ 116, -106, 119 },
|
| 131 |
+
{ -51, 117, 60 },
|
| 132 |
+
{ -40, 124, 122 },
|
| 133 |
+
{ 114, -118, -124 },
|
| 134 |
+
{ -115, 55, 13 },
|
| 135 |
+
{ -124, 4, -76 },
|
| 136 |
+
{ -124, 26, 64 },
|
| 137 |
+
{ -74, 117, 8 },
|
| 138 |
+
{ -119, 59, -46 },
|
| 139 |
+
{ 110, 120, -115 },
|
| 140 |
+
{ 121, 116, -38 },
|
| 141 |
+
{ -115, 114, -106 },
|
| 142 |
+
{ -122, 121, -25 },
|
| 143 |
+
};
|
| 144 |
+
|
| 145 |
+
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut0_lo[64] = { -24, -37, -41, 23, 29, -3, -59, -68, 11, -84, 35, -19, 54, 12, -41, 16, -62, 69, -65, 70, 75, 55, -6, -47, 73, -22, 62, 5, -123, -119, -24, -57, -125, -30, -19, 43, -56, 52, 120, 123, 124, 127, -97, 119, 117, 7, 9, -122, -123, 5, 51, 100, 114, 62, -56, -122, -123, -121, 122, -127, 114, 121, -114, -119 };
|
| 146 |
+
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut0_hi[64] = { 5, 16, 10, -18, 18, -13, -37, 29, -15, -15, -19, -75, 31, 60, -15, 1, 20, 43, 66, 63, 4, 63, -115, -108, 53, -54, 58, -62, -5, -66, -24, -31, -61, -63, 30, 35, 118, -76, 56, 118, 122, 115, 122, 38, -113, 124, 121, 63, 25, -47, -24, 116, -51, -40, 114, -115, -124, -124, -74, -119, 110, 121, -115, -122 };
|
| 147 |
+
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut1_lo[64] = { -19, -20, -18, -6, -15, 4, -25, -13, -54, -5, -51, 8, 9, -74, -73, 46, -70, -14, -73, 11, -2, 43, 62, 32, -64, 67, 61, 80, -46, -58, -120, 48, -52, 77, -126, -118, 69, -122, 6, -22, -26, 4, -121, 46, 54, 117, 120, -113, -114, 124, 122, -123, -111, 123, 119, 6, 8, 40, -120, 1, 114, 114, 97, 116 };
|
| 148 |
+
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut1_hi[64] = { -19, -14, 8, -41, -35, -51, -47, 16, 8, 14, -71, -42, -64, -2, 37, 49, -81, 42, -41, -53, 58, -62, -30, -47, 59, 31, 65, 19, -119, 13, -121, -120, -113, 46, -121, -124, -18, -122, -122, 26, -48, 44, -56, 112, -115, -70, 55, 110, 123, 113, 124, -106, 117, 124, -118, 55, 4, 26, 117, 59, 120, 116, 114, 121 };
|
| 149 |
+
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut2_lo[64] = { -24, -123, 18, 27, -65, 0, 72, -66, -122, 30, 51, 50, 62, -55, -34, 19, -111, 116, 20, 12, -40, -65, -121, 16, 65, 121, -9, -18, 55, -56, -122, -72, -3, 27, -11, -118, -22, 119, -116, -59, 115, -10, -34, -48, -115, 76, 26, 102, 19, -58, -8, 51, -62, 52, -59, -31, 120, -121, -3, 15, 111, 28, 118, 48 };
|
| 150 |
+
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut2_hi[64] = { 75, 126, -122, -70, -16, 18, 123, -25, -63, 120, 69, -18, 121, -118, -26, 66, 0, 120, 8, -98, -67, -38, -124, 118, -124, -125, 56, 123, 115, -23, 38, -66, 118, 65, 48, -54, 34, 46, -4, 62, -117, 120, -12, 124, -109, 44, 14, -59, -120, -121, -9, 119, 60, 122, -124, 13, -76, 64, 8, -46, -115, -38, -106, -25 };
|
| 151 |
+
|
| 152 |
+
GGML_VQ3_ALIGN64 static const uint8_t ggml_vq3_perm_rep64[64] = {
|
| 153 |
+
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,
|
| 154 |
+
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,
|
| 155 |
+
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19
|
| 156 |
+
};
|
| 157 |
+
|
| 158 |
+
GGML_VQ3_ALIGN64 static const uint8_t ggml_vq3_shuf_sc_gather[64] = {
|
| 159 |
+
0, 1, 2,63, 3, 4, 5,63, 6, 7, 8,63, 9,10,11,63,
|
| 160 |
+
12,13,14,63, 15,16,17,63, 18,19,20,63, 21,63,20,21,
|
| 161 |
+
63,63,63,63, 63,63,63,63, 63,63,63,63, 63,63,63,63,
|
| 162 |
+
63,63,63,63, 63,63,63,63, 63,63,63,63, 63,63,63,63
|
| 163 |
+
};
|
| 164 |
+
|
| 165 |
+
GGML_VQ3_ALIGN64 static const uint16_t ggml_vq3_sc_rshift[32] = {
|
| 166 |
+
7,7, 7,7, 7,7, 7,7, 7,7, 7,7, 7,7, 7,6,
|
| 167 |
+
0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0
|
| 168 |
+
};
|
| 169 |
+
|
| 170 |
+
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_weights_sc[64] = {
|
| 171 |
+
1,2,4,0, 1,2,4,0, 1,2,4,0, 1,2,4,0,
|
| 172 |
+
1,2,4,0, 1,2,4,0, 1,2,4,0, 1,0,2,4,
|
| 173 |
+
0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
|
| 174 |
+
0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
|
| 175 |
+
};
|
| 176 |
+
|
| 177 |
+
GGML_VQ3_ALIGN64 static const uint16_t ggml_vq3_shuf_mult_i16[32] = {
|
| 178 |
+
0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6,
|
| 179 |
+
8, 8, 8, 8, 10,10,10,10, 12,12,12,12, 14,14,14,14
|
| 180 |
+
};
|
| 181 |
+
|
| 182 |
+
#if defined(__AVX512F__) || (defined(__BMI2__) && (defined(__x86_64__) || defined(_M_X64)))
|
| 183 |
+
#include <immintrin.h>
|
| 184 |
+
#endif
|
| 185 |
+
#if defined(__AVX512F__)
|
| 186 |
+
#if defined(__GNUC__) || defined(__clang__)
|
| 187 |
+
static inline __m512i ggml_vq3_loadu_si512(const void * p) {
|
| 188 |
+
__m512i r;
|
| 189 |
+
__asm__("vmovdqu64 %1, %0" : "=v"(r) : "m"(*(const uint64_t *)p));
|
| 190 |
+
return r;
|
| 191 |
+
}
|
| 192 |
+
#else
|
| 193 |
+
#define ggml_vq3_loadu_si512(p) _mm512_loadu_si512((const __m512i *)(p))
|
| 194 |
+
#endif
|
| 195 |
+
#endif
|
| 196 |
+
|
| 197 |
+
static inline uint64_t ggml_vq3_unpack_scales(uint32_t s24) {
|
| 198 |
+
#if defined(__BMI2__) && (defined(__x86_64__) || defined(_M_X64))
|
| 199 |
+
return _pdep_u64((uint64_t)s24, 0x0707070707070707ULL) + 0x0202020202020202ULL;
|
| 200 |
+
#else
|
| 201 |
+
uint64_t m_all = 0x0202020202020202ULL;
|
| 202 |
+
for (int b = 0; b < 8; ++b) {
|
| 203 |
+
m_all += ((uint64_t)((s24 >> (3 * b)) & 0x7U)) << (8 * b);
|
| 204 |
+
}
|
| 205 |
+
return m_all;
|
| 206 |
+
#endif
|
| 207 |
+
}
|
| 208 |
+
|
| 209 |
+
GGML_VQ3_ALIGN64 static const uint32_t ggml_vq3_shuf_mult512[16] = {
|
| 210 |
+
0x80008000, 0x80008000, 0x80018001, 0x80018001,
|
| 211 |
+
0x80028002, 0x80028002, 0x80038003, 0x80038003,
|
| 212 |
+
0x80048004, 0x80048004, 0x80058005, 0x80058005,
|
| 213 |
+
0x80068006, 0x80068006, 0x80078007, 0x80078007
|
| 214 |
+
};
|
| 215 |
+
|
| 216 |
+
GGML_VQ3_ALIGN64 static const uint32_t ggml_q2_lut_abs512[16] = {
|
| 217 |
+
0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401,
|
| 218 |
+
0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401,
|
| 219 |
+
0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401,
|
| 220 |
+
0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401
|
| 221 |
+
};
|
| 222 |
+
|
| 223 |
+
GGML_VQ3_ALIGN64 static const uint32_t ggml_q2_shift_qs[16] = {
|
| 224 |
+
0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6
|
| 225 |
+
};
|
| 226 |
+
|
| 227 |
+
GGML_VQ3_ALIGN64 static const uint64_t ggml_q2_ror_qh[8] = {
|
| 228 |
+
62, 63, 0, 1, 2, 3, 4, 5
|
| 229 |
+
};
|
| 230 |
+
|
| 231 |
+
#endif // GGML_VQ3_CODEBOOK_H
|
vq3/vq3_enc_lib.c
ADDED
|
@@ -0,0 +1,369 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/*
|
| 2 |
+
* Self-contained OpenMP AVX-512 LUT-VQ3-64 (GGML_TYPE_VQ3_0 = 43) encoder & transcoder shared library.
|
| 3 |
+
* Build:
|
| 4 |
+
* gcc -O3 -march=native -mavx512f -mf16c -fopenmp -shared -fPIC vq3/vq3_enc_lib.c -lm -o vq3/libvq3_enc.so
|
| 5 |
+
*/
|
| 6 |
+
#include <stdio.h>
|
| 7 |
+
#include <stdlib.h>
|
| 8 |
+
#include <stdint.h>
|
| 9 |
+
#include <string.h>
|
| 10 |
+
#include <math.h>
|
| 11 |
+
#include <omp.h>
|
| 12 |
+
#include <immintrin.h>
|
| 13 |
+
#include "vq3_codebook.h"
|
| 14 |
+
|
| 15 |
+
#define QK_VQ3_0 64
|
| 16 |
+
typedef uint16_t ggml_half;
|
| 17 |
+
typedef struct { ggml_half d; uint8_t qs[22]; } block_vq3_0;
|
| 18 |
+
typedef struct {
|
| 19 |
+
ggml_half d;
|
| 20 |
+
uint8_t scales[4];
|
| 21 |
+
uint8_t qs[16];
|
| 22 |
+
uint8_t qh[8];
|
| 23 |
+
} block_q2_0;
|
| 24 |
+
|
| 25 |
+
static inline float fp16_to_fp32(ggml_half h) { return _cvtsh_ss(h); }
|
| 26 |
+
static inline ggml_half fp32_to_fp16(float f) { return _cvtss_sh(f, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); }
|
| 27 |
+
static inline float bf16_to_fp32(uint16_t b) {
|
| 28 |
+
uint32_t u = ((uint32_t)b) << 16;
|
| 29 |
+
float f;
|
| 30 |
+
memcpy(&f, &u, sizeof(f));
|
| 31 |
+
return f;
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
static float g_cb_q0[128] __attribute__((aligned(64)));
|
| 35 |
+
static float g_cb_q1[128] __attribute__((aligned(64)));
|
| 36 |
+
static float g_cb_q2[128] __attribute__((aligned(64)));
|
| 37 |
+
static float g_cb_c20[128] __attribute__((aligned(64)));
|
| 38 |
+
static float g_cb_c21[128] __attribute__((aligned(64)));
|
| 39 |
+
static float g_cb_c22[128] __attribute__((aligned(64)));
|
| 40 |
+
static int g_cb_init = 0;
|
| 41 |
+
|
| 42 |
+
static void init_vq3_enc_tables(void) {
|
| 43 |
+
if (g_cb_init) return;
|
| 44 |
+
for (int c = 0; c < 128; ++c) {
|
| 45 |
+
float v0 = (float)ggml_vq3_cb[c][0] * GGML_VQ3_G;
|
| 46 |
+
float v1 = (float)ggml_vq3_cb[c][1] * GGML_VQ3_G;
|
| 47 |
+
float v2 = (float)ggml_vq3_cb[c][2] * GGML_VQ3_G;
|
| 48 |
+
g_cb_q0[c] = v0; g_cb_c20[c] = v0 * v0;
|
| 49 |
+
g_cb_q1[c] = v1; g_cb_c21[c] = v1 * v1;
|
| 50 |
+
g_cb_q2[c] = v2; g_cb_c22[c] = v2 * v2;
|
| 51 |
+
}
|
| 52 |
+
g_cb_init = 1;
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
void quantize_row_vq3_0_fast(const float * restrict x, block_vq3_0 * restrict y, int64_t k) {
|
| 56 |
+
const int64_t nb = k / QK_VQ3_0;
|
| 57 |
+
const __m512i v_iota = _mm512_setr_epi32(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15);
|
| 58 |
+
const __m512i v_step = _mm512_set1_epi32(16);
|
| 59 |
+
const __m512 v_neg2 = _mm512_set1_ps(-2.0f);
|
| 60 |
+
|
| 61 |
+
for (int64_t b = 0; b < nb; ++b) {
|
| 62 |
+
const float * xb = x + b * 64;
|
| 63 |
+
float sub_amax[8];
|
| 64 |
+
float blk_amax = 0.0f;
|
| 65 |
+
for (int s = 0; s < 8; ++s) {
|
| 66 |
+
float amax = 0.0f;
|
| 67 |
+
for (int j = 0; j < 8; ++j) {
|
| 68 |
+
float av = fabsf(xb[s * 8 + j]);
|
| 69 |
+
if (av > amax) amax = av;
|
| 70 |
+
}
|
| 71 |
+
sub_amax[s] = amax * 0.88f;
|
| 72 |
+
if (sub_amax[s] > blk_amax) blk_amax = sub_amax[s];
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
ggml_half d_h = (blk_amax > 0.0f) ? fp32_to_fp16(blk_amax / 9.0f) : 0;
|
| 76 |
+
float d = fp16_to_fp32(d_h);
|
| 77 |
+
if (d <= 1e-30f) {
|
| 78 |
+
y[b].d = 0;
|
| 79 |
+
memset(y[b].qs, 0, 22);
|
| 80 |
+
continue;
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
float m[8];
|
| 84 |
+
for (int s = 0; s < 8; ++s) {
|
| 85 |
+
float mr = roundf(sub_amax[s] / d);
|
| 86 |
+
if (mr < 2.0f) mr = 2.0f;
|
| 87 |
+
if (mr > 9.0f) mr = 9.0f;
|
| 88 |
+
m[s] = mr;
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
uint8_t idx22[22];
|
| 92 |
+
float w_rec[64];
|
| 93 |
+
|
| 94 |
+
for (int iter = 0; iter < 2; ++iter) {
|
| 95 |
+
float eff_s[64];
|
| 96 |
+
for (int s = 0; s < 8; ++s) {
|
| 97 |
+
float es = d * m[s];
|
| 98 |
+
for (int j = 0; j < 8; ++j) eff_s[s * 8 + j] = es;
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
for (int j = 0; j < 20; ++j) {
|
| 102 |
+
float w0 = xb[j], s0 = eff_s[j];
|
| 103 |
+
float w1 = xb[j + 22], s1 = eff_s[j + 22];
|
| 104 |
+
float w2 = xb[j + 44], s2 = eff_s[j + 44];
|
| 105 |
+
__m512 vs02 = _mm512_set1_ps(s0 * s0);
|
| 106 |
+
__m512 vs12 = _mm512_set1_ps(s1 * s1);
|
| 107 |
+
__m512 vs22 = _mm512_set1_ps(s2 * s2);
|
| 108 |
+
__m512 vws0 = _mm512_mul_ps(v_neg2, _mm512_set1_ps(w0 * s0));
|
| 109 |
+
__m512 vws1 = _mm512_mul_ps(v_neg2, _mm512_set1_ps(w1 * s1));
|
| 110 |
+
__m512 vws2 = _mm512_mul_ps(v_neg2, _mm512_set1_ps(w2 * s2));
|
| 111 |
+
|
| 112 |
+
__m512 best_d = _mm512_set1_ps(1e30f);
|
| 113 |
+
__m512i best_i = _mm512_setzero_si512();
|
| 114 |
+
__m512i cur_i = v_iota;
|
| 115 |
+
|
| 116 |
+
for (int c = 0; c < 128; c += 16) {
|
| 117 |
+
__m512 dist = _mm512_mul_ps(vs02, _mm512_load_ps(g_cb_c20 + c));
|
| 118 |
+
dist = _mm512_fmadd_ps(vs12, _mm512_load_ps(g_cb_c21 + c), dist);
|
| 119 |
+
dist = _mm512_fmadd_ps(vs22, _mm512_load_ps(g_cb_c22 + c), dist);
|
| 120 |
+
dist = _mm512_fmadd_ps(vws0, _mm512_load_ps(g_cb_q0 + c), dist);
|
| 121 |
+
dist = _mm512_fmadd_ps(vws1, _mm512_load_ps(g_cb_q1 + c), dist);
|
| 122 |
+
dist = _mm512_fmadd_ps(vws2, _mm512_load_ps(g_cb_q2 + c), dist);
|
| 123 |
+
__mmask16 lt = _mm512_cmp_ps_mask(dist, best_d, _CMP_LT_OQ);
|
| 124 |
+
best_d = _mm512_mask_blend_ps(lt, best_d, dist);
|
| 125 |
+
best_i = _mm512_mask_blend_epi32(lt, best_i, cur_i);
|
| 126 |
+
cur_i = _mm512_add_epi32(cur_i, v_step);
|
| 127 |
+
}
|
| 128 |
+
float min_val = _mm512_reduce_min_ps(best_d);
|
| 129 |
+
__mmask16 eq = _mm512_cmp_ps_mask(best_d, _mm512_set1_ps(min_val), _CMP_EQ_OQ);
|
| 130 |
+
int lane = __builtin_ctz((unsigned int)eq);
|
| 131 |
+
int align_idx[16];
|
| 132 |
+
_mm512_storeu_si512((__m512i *)align_idx, best_i);
|
| 133 |
+
int best_c = align_idx[lane];
|
| 134 |
+
idx22[j] = (uint8_t)best_c;
|
| 135 |
+
w_rec[j] = g_cb_q0[best_c];
|
| 136 |
+
w_rec[j + 22] = g_cb_q1[best_c];
|
| 137 |
+
w_rec[j + 44] = g_cb_q2[best_c];
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
int s7_u3 = ((int)roundf(m[7] - 2.0f)) & 0x7;
|
| 141 |
+
for (int u = 0; u < 2; ++u) {
|
| 142 |
+
int j = 20 + u;
|
| 143 |
+
int bit_half = (s7_u3 >> (u + 1)) & 1;
|
| 144 |
+
int c_start = bit_half ? 64 : 0;
|
| 145 |
+
float w0 = xb[j], s0 = eff_s[j];
|
| 146 |
+
float w1 = xb[j + 22], s1 = eff_s[j + 22];
|
| 147 |
+
__m512 vs02 = _mm512_set1_ps(s0 * s0);
|
| 148 |
+
__m512 vs12 = _mm512_set1_ps(s1 * s1);
|
| 149 |
+
__m512 vws0 = _mm512_mul_ps(v_neg2, _mm512_set1_ps(w0 * s0));
|
| 150 |
+
__m512 vws1 = _mm512_mul_ps(v_neg2, _mm512_set1_ps(w1 * s1));
|
| 151 |
+
__m512 best_d = _mm512_set1_ps(1e30f);
|
| 152 |
+
__m512i best_i = _mm512_setzero_si512();
|
| 153 |
+
__m512i cur_i = _mm512_add_epi32(v_iota, _mm512_set1_epi32(c_start));
|
| 154 |
+
for (int c = c_start; c < c_start + 64; c += 16) {
|
| 155 |
+
__m512 dist = _mm512_mul_ps(vs02, _mm512_load_ps(g_cb_c20 + c));
|
| 156 |
+
dist = _mm512_fmadd_ps(vs12, _mm512_load_ps(g_cb_c21 + c), dist);
|
| 157 |
+
dist = _mm512_fmadd_ps(vws0, _mm512_load_ps(g_cb_q0 + c), dist);
|
| 158 |
+
dist = _mm512_fmadd_ps(vws1, _mm512_load_ps(g_cb_q1 + c), dist);
|
| 159 |
+
__mmask16 lt = _mm512_cmp_ps_mask(dist, best_d, _CMP_LT_OQ);
|
| 160 |
+
best_d = _mm512_mask_blend_ps(lt, best_d, dist);
|
| 161 |
+
best_i = _mm512_mask_blend_epi32(lt, best_i, cur_i);
|
| 162 |
+
cur_i = _mm512_add_epi32(cur_i, v_step);
|
| 163 |
+
}
|
| 164 |
+
float min_val = _mm512_reduce_min_ps(best_d);
|
| 165 |
+
__mmask16 eq = _mm512_cmp_ps_mask(best_d, _mm512_set1_ps(min_val), _CMP_EQ_OQ);
|
| 166 |
+
int lane = __builtin_ctz((unsigned int)eq);
|
| 167 |
+
int align_idx[16];
|
| 168 |
+
_mm512_storeu_si512((__m512i *)align_idx, best_i);
|
| 169 |
+
int best_c = align_idx[lane];
|
| 170 |
+
idx22[j] = (uint8_t)best_c;
|
| 171 |
+
w_rec[j] = g_cb_q0[best_c];
|
| 172 |
+
w_rec[j + 22] = g_cb_q1[best_c];
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
+
if (iter == 0) {
|
| 176 |
+
for (int s = 0; s < 8; ++s) {
|
| 177 |
+
float wq = 0.0f, qq = 0.0f;
|
| 178 |
+
for (int j = 0; j < 8; ++j) {
|
| 179 |
+
float wv = xb[s * 8 + j];
|
| 180 |
+
float qv = w_rec[s * 8 + j];
|
| 181 |
+
wq += wv * qv;
|
| 182 |
+
qq += qv * qv;
|
| 183 |
+
}
|
| 184 |
+
if (qq > 0.0f) {
|
| 185 |
+
float mo = roundf(wq / fmaxf(d * qq, 1e-30f));
|
| 186 |
+
if (mo < 2.0f) mo = 2.0f;
|
| 187 |
+
if (mo > 9.0f) mo = 9.0f;
|
| 188 |
+
m[s] = mo;
|
| 189 |
+
}
|
| 190 |
+
}
|
| 191 |
+
}
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
uint32_t s24 = 0;
|
| 195 |
+
for (int s = 0; s < 8; ++s) {
|
| 196 |
+
uint32_t sv = ((uint32_t)roundf(m[s] - 2.0f)) & 0x7U;
|
| 197 |
+
s24 |= (sv << (3 * s));
|
| 198 |
+
}
|
| 199 |
+
y[b].d = d_h;
|
| 200 |
+
for (int k_i = 0; k_i < 22; ++k_i) {
|
| 201 |
+
uint8_t msb = (uint8_t)((s24 >> k_i) & 1U);
|
| 202 |
+
y[b].qs[k_i] = (uint8_t)((idx22[k_i] & 0x7FU) | (msb << 7));
|
| 203 |
+
}
|
| 204 |
+
}
|
| 205 |
+
}
|
| 206 |
+
|
| 207 |
+
void quantize_bf16_to_vq3_0_omp(const uint16_t * src_bf16, void * dst_vq3, int64_t nrow, int64_t ncol) {
|
| 208 |
+
init_vq3_enc_tables();
|
| 209 |
+
block_vq3_0 * y = (block_vq3_0 *)dst_vq3;
|
| 210 |
+
const int64_t nblk_row = ncol / 64;
|
| 211 |
+
#pragma omp parallel
|
| 212 |
+
{
|
| 213 |
+
float * row_f32 = (float *)malloc((size_t)ncol * sizeof(float));
|
| 214 |
+
if (!row_f32) { fprintf(stderr, "OOM in transcode_any_to_vq3_0_omp\n"); abort(); }
|
| 215 |
+
#pragma omp for schedule(static)
|
| 216 |
+
for (int64_t r = 0; r < nrow; ++r) {
|
| 217 |
+
const uint16_t * sr = src_bf16 + r * ncol;
|
| 218 |
+
for (int64_t c = 0; c < ncol; ++c) row_f32[c] = bf16_to_fp32(sr[c]);
|
| 219 |
+
quantize_row_vq3_0_fast(row_f32, y + r * nblk_row, ncol);
|
| 220 |
+
}
|
| 221 |
+
free(row_f32);
|
| 222 |
+
}
|
| 223 |
+
}
|
| 224 |
+
|
| 225 |
+
void dequantize_row_q2_0_local(const block_q2_0 * restrict x, float * restrict y, int64_t k) {
|
| 226 |
+
static const int8_t kvalues_fractal3[8] = { -14, -8, -4, -1, 1, 4, 8, 14 };
|
| 227 |
+
const int64_t nb = k / 64;
|
| 228 |
+
for (int64_t i = 0; i < nb; i++) {
|
| 229 |
+
const float d = fp16_to_fp32(x[i].d);
|
| 230 |
+
float eff_s[8];
|
| 231 |
+
for (int b = 0; b < 4; ++b) {
|
| 232 |
+
eff_s[b] = d * (float)(4 + (x[i].scales[b] & 0x0F));
|
| 233 |
+
eff_s[b + 4] = d * (float)(4 + ((x[i].scales[b] >> 4) & 0x0F));
|
| 234 |
+
}
|
| 235 |
+
for (int g = 0; g < 4; ++g) {
|
| 236 |
+
for (int k_idx = 0; k_idx < 16; ++k_idx) {
|
| 237 |
+
const int j = g * 16 + k_idx;
|
| 238 |
+
const uint8_t lo2 = (x[i].qs[k_idx] >> (2 * g)) & 0x03;
|
| 239 |
+
const uint8_t hi1 = (x[i].qh[k_idx & 7] >> (2 * g + (k_idx >> 3))) & 0x01;
|
| 240 |
+
const uint8_t idx = (uint8_t)(lo2 | (hi1 << 2));
|
| 241 |
+
y[i * 64 + j] = eff_s[j >> 3] * (float)kvalues_fractal3[idx];
|
| 242 |
+
}
|
| 243 |
+
}
|
| 244 |
+
}
|
| 245 |
+
}
|
| 246 |
+
|
| 247 |
+
void transcode_q2_0_to_vq3_0_omp(const void * src_q2_0, void * dst_vq3, int64_t nrow, int64_t ncol) {
|
| 248 |
+
init_vq3_enc_tables();
|
| 249 |
+
const block_q2_0 * x = (const block_q2_0 *)src_q2_0;
|
| 250 |
+
block_vq3_0 * y = (block_vq3_0 *)dst_vq3;
|
| 251 |
+
const int64_t nblk_row = ncol / 64;
|
| 252 |
+
#pragma omp parallel
|
| 253 |
+
{
|
| 254 |
+
float * row_f32 = (float *)malloc((size_t)ncol * sizeof(float));
|
| 255 |
+
if (!row_f32) { fprintf(stderr, "OOM in transcode_q2_0_to_vq3_0_omp\n"); abort(); }
|
| 256 |
+
#pragma omp for schedule(static)
|
| 257 |
+
for (int64_t r = 0; r < nrow; ++r) {
|
| 258 |
+
dequantize_row_q2_0_local(x + r * nblk_row, row_f32, ncol);
|
| 259 |
+
quantize_row_vq3_0_fast(row_f32, y + r * nblk_row, ncol);
|
| 260 |
+
}
|
| 261 |
+
free(row_f32);
|
| 262 |
+
}
|
| 263 |
+
}
|
| 264 |
+
|
| 265 |
+
typedef struct {
|
| 266 |
+
ggml_half d;
|
| 267 |
+
ggml_half dmin;
|
| 268 |
+
uint8_t scales[12];
|
| 269 |
+
uint8_t qh[32];
|
| 270 |
+
uint8_t qs[128];
|
| 271 |
+
} block_q5_K_local;
|
| 272 |
+
|
| 273 |
+
typedef struct {
|
| 274 |
+
uint8_t ql[128];
|
| 275 |
+
uint8_t qh[64];
|
| 276 |
+
int8_t scales[16];
|
| 277 |
+
ggml_half d;
|
| 278 |
+
} block_q6_K_local;
|
| 279 |
+
|
| 280 |
+
static inline void get_scale_min_k4_local(int j, const uint8_t * restrict q, uint8_t * restrict d, uint8_t * restrict m) {
|
| 281 |
+
if (j < 4) {
|
| 282 |
+
*d = q[j] & 63;
|
| 283 |
+
*m = q[j + 4] & 63;
|
| 284 |
+
} else {
|
| 285 |
+
*d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4);
|
| 286 |
+
*m = (q[j + 4] >> 4) | ((q[j - 0] >> 6) << 4);
|
| 287 |
+
}
|
| 288 |
+
}
|
| 289 |
+
|
| 290 |
+
static void dequantize_row_q5_K_local(const block_q5_K_local * restrict x, float * restrict y, int64_t k) {
|
| 291 |
+
const int64_t nb = k / 256;
|
| 292 |
+
for (int64_t i = 0; i < nb; i++) {
|
| 293 |
+
const uint8_t * ql = x[i].qs;
|
| 294 |
+
const uint8_t * qh = x[i].qh;
|
| 295 |
+
const float d = fp16_to_fp32(x[i].d);
|
| 296 |
+
const float min = fp16_to_fp32(x[i].dmin);
|
| 297 |
+
int is = 0;
|
| 298 |
+
uint8_t sc, m;
|
| 299 |
+
uint8_t u1 = 1, u2 = 2;
|
| 300 |
+
for (int j = 0; j < 256; j += 64) {
|
| 301 |
+
get_scale_min_k4_local(is + 0, x[i].scales, &sc, &m);
|
| 302 |
+
const float d1 = d * sc;
|
| 303 |
+
const float m1 = min * m;
|
| 304 |
+
get_scale_min_k4_local(is + 1, x[i].scales, &sc, &m);
|
| 305 |
+
const float d2 = d * sc;
|
| 306 |
+
const float m2 = min * m;
|
| 307 |
+
for (int l = 0; l < 32; ++l) *y++ = d1 * ((ql[l] & 0xF) + (qh[l] & u1 ? 16 : 0)) - m1;
|
| 308 |
+
for (int l = 0; l < 32; ++l) *y++ = d2 * ((ql[l] >> 4) + (qh[l] & u2 ? 16 : 0)) - m2;
|
| 309 |
+
ql += 32;
|
| 310 |
+
is += 2;
|
| 311 |
+
u1 <<= 2;
|
| 312 |
+
u2 <<= 2;
|
| 313 |
+
}
|
| 314 |
+
}
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
static void dequantize_row_q6_K_local(const block_q6_K_local * restrict x, float * restrict y, int64_t k) {
|
| 318 |
+
const int64_t nb = k / 256;
|
| 319 |
+
for (int64_t i = 0; i < nb; i++) {
|
| 320 |
+
const float d = fp16_to_fp32(x[i].d);
|
| 321 |
+
const uint8_t * restrict ql = x[i].ql;
|
| 322 |
+
const uint8_t * restrict qh = x[i].qh;
|
| 323 |
+
const int8_t * restrict sc = x[i].scales;
|
| 324 |
+
for (int n = 0; n < 256; n += 128) {
|
| 325 |
+
for (int l = 0; l < 32; ++l) {
|
| 326 |
+
int is = l / 16;
|
| 327 |
+
const int8_t q1 = (int8_t)((ql[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32;
|
| 328 |
+
const int8_t q2 = (int8_t)((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32;
|
| 329 |
+
const int8_t q3 = (int8_t)((ql[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32;
|
| 330 |
+
const int8_t q4 = (int8_t)((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32;
|
| 331 |
+
y[l + 0] = d * sc[is + 0] * q1;
|
| 332 |
+
y[l + 32] = d * sc[is + 2] * q2;
|
| 333 |
+
y[l + 64] = d * sc[is + 4] * q3;
|
| 334 |
+
y[l + 96] = d * sc[is + 6] * q4;
|
| 335 |
+
}
|
| 336 |
+
y += 128;
|
| 337 |
+
ql += 64;
|
| 338 |
+
qh += 32;
|
| 339 |
+
sc += 8;
|
| 340 |
+
}
|
| 341 |
+
}
|
| 342 |
+
}
|
| 343 |
+
|
| 344 |
+
void transcode_any_to_vq3_0_omp(const void * src, int orig_type, void * dst_vq3, int64_t nrow, int64_t ncol) {
|
| 345 |
+
init_vq3_enc_tables();
|
| 346 |
+
const uint8_t * x = (const uint8_t *)src;
|
| 347 |
+
block_vq3_0 * y = (block_vq3_0 *)dst_vq3;
|
| 348 |
+
const int64_t nblk_out = ncol / 64;
|
| 349 |
+
size_t row_bytes_in = 0;
|
| 350 |
+
if (orig_type == 42) row_bytes_in = (size_t)(ncol / 64) * 30;
|
| 351 |
+
else if (orig_type == 13) row_bytes_in = (size_t)(ncol / 256) * 176;
|
| 352 |
+
else if (orig_type == 14) row_bytes_in = (size_t)(ncol / 256) * 210;
|
| 353 |
+
else { fprintf(stderr, "Unsupported orig_type=%d\n", orig_type); abort(); }
|
| 354 |
+
|
| 355 |
+
#pragma omp parallel
|
| 356 |
+
{
|
| 357 |
+
float * row_f32 = (float *)malloc((size_t)ncol * sizeof(float));
|
| 358 |
+
if (!row_f32) { fprintf(stderr, "OOM in transcode_any_to_vq3_0_omp\n"); abort(); }
|
| 359 |
+
#pragma omp for schedule(static)
|
| 360 |
+
for (int64_t r = 0; r < nrow; ++r) {
|
| 361 |
+
const void * sr = x + (size_t)r * row_bytes_in;
|
| 362 |
+
if (orig_type == 42) dequantize_row_q2_0_local((const block_q2_0 *)sr, row_f32, ncol);
|
| 363 |
+
else if (orig_type == 13) dequantize_row_q5_K_local((const block_q5_K_local *)sr, row_f32, ncol);
|
| 364 |
+
else if (orig_type == 14) dequantize_row_q6_K_local((const block_q6_K_local *)sr, row_f32, ncol);
|
| 365 |
+
quantize_row_vq3_0_fast(row_f32, y + r * nblk_out, ncol);
|
| 366 |
+
}
|
| 367 |
+
free(row_f32);
|
| 368 |
+
}
|
| 369 |
+
}
|