lingtiny30vq3 / build_and_run.sh
HelloSun's picture
LUT-VQ3-64: machine-tuned (threads/NUMA/ubatch), full 128k ctx data, one-click build, results
511ba50 verified
Raw History Blame Contribute Delete
6.66 kB
#!/usr/bin/env bash
# =============================================================================
# build_and_run.sh — LUT-VQ3-64 for Ling-3.0-tiny:一鍵編譯 + 驗證 + 下載 + 實驗
#
# bash build_and_run.sh # 編譯 + 單元測試
# bash build_and_run.sh --download # 另外下載 VQ3 GGUF 並做推論冒煙測試
# bash build_and_run.sh --experiments # 另外跑 PPL + llama-bench + 生成樣本
# bash build_and_run.sh --ctx128k # 另外跑 128k 上下文實驗(prefill/decode/RSS)
# bash build_and_run.sh --all # 全部
#
# 需求:git, cmake>=3.14, gcc/g++(x86-64 建議 AVX-512;VQ3 LUT 核心需 VBMI),
# python3(僅重量化時需要)
# 模型:inclusionAI/Ling-3.0-tiny(MIT)之 LUT-VQ3-64 量化(專家張量 3.00 bpw)
#
# ※ 本機最佳化(2×40C Xeon,雙 NUMA)實測結論:
# - decode(tg)用 -t 32 最快(38.9 t/s);prefill(pp)用 -t 64 最快(353+ t/s)
# - 故預設採用 -t 32 -tb 64(decode 32 執行緒 / prefill 64 執行緒)
# - 【不要】加 --numa distribute:實測 tg@t80 崩至 0.9 t/s、pp@t48 −21%
# - 80 執行緒會跨 NUMA 抖動(pp 崩至 ~70 t/s),避免使用
# =============================================================================
set -euo pipefail
LLAMA_REPO="https://github.com/ggml-org/llama.cpp"
LLAMA_COMMIT="61dbf49228c8a96b72e40c89d93cb7c003411f8c"
HF_REPO="HelloSun/lingtiny30vq3"
MODEL_NAME="Ling-3.0-tiny-VQ3.gguf"
MODEL_URL="https://huggingface.co/${HF_REPO}/resolve/main/${MODEL_NAME}"
HERE="$(cd "$(dirname "$0")" && pwd)"
WORK="${HERE}/work"
JOBS="$(nproc)"
T_DEC=32 # decode 最佳執行緒數(本機實測)
T_BATCH=64 # prefill/batch 最佳執行緒數(本機實測)
DO_DOWNLOAD=0; DO_EXPERIMENTS=0; DO_CTX128K=0
while [[ $# -gt 0 ]]; do
case "$1" in
--jobs) JOBS="$2"; shift 2;;
--download) DO_DOWNLOAD=1; shift;;
--experiments) DO_EXPERIMENTS=1; shift;;
--ctx128k) DO_CTX128K=1; shift;;
--all) DO_DOWNLOAD=1; DO_EXPERIMENTS=1; DO_CTX128K=1; shift;;
*) echo "unknown arg: $1"; exit 1;;
esac
done
echo "== [1/6] 取得 llama.cpp @ ${LLAMA_COMMIT:0:9} =="
mkdir -p "${WORK}"
if [[ ! -d "${WORK}/llama.cpp" ]]; then
git clone "${LLAMA_REPO}" "${WORK}/llama.cpp"
fi
git -C "${WORK}/llama.cpp" fetch --depth 1 origin "${LLAMA_COMMIT}" 2>/dev/null || true
git -C "${WORK}/llama.cpp" checkout -q --detach "${LLAMA_COMMIT}"
echo "== [2/6] 套用 LUT-VQ3-64 補丁 =="
if git -C "${WORK}/llama.cpp" apply --check "${HERE}/patches/0001-lut-vq3-64-avx512-vbmi-port-to-61dbf492.patch" 2>/dev/null; then
git -C "${WORK}/llama.cpp" apply "${HERE}/patches/0001-lut-vq3-64-avx512-vbmi-port-to-61dbf492.patch"
echo "patch applied"
elif grep -rq "GGML_TYPE_VQ3_0" "${WORK}/llama.cpp/ggml/include/ggml.h"; then
echo "patch already applied"
else
echo "ERROR: patch does not apply"; exit 1
fi
echo "== [3/6] 編譯(-j ${JOBS},GGML_NATIVE=ON:AVX-512/AMX 自動啟用)=="
cmake -S "${WORK}/llama.cpp" -B "${WORK}/llama.cpp/build" \
-DGGML_NATIVE=ON -DLLAMA_CURL=OFF -DBUILD_SHARED_LIBS=OFF \
-DCMAKE_BUILD_TYPE=Release
cmake --build "${WORK}/llama.cpp/build" -j "${JOBS}" \
--target llama-cli llama-server llama-quantize llama-perplexity llama-bench
BIN="${WORK}/llama.cpp/build/bin"
echo "== [4/6] 編譯 VQ3 編碼器函式庫(量化用)=="
mkdir -p "${WORK}/vq3"
cp "${HERE}/vq3/vq3_enc_lib.c" "${HERE}/vq3/vq3_codebook.h" "${WORK}/vq3/"
gcc -O3 -march=native -mavx512f -mf16c -fopenmp -shared -fPIC \
"${WORK}/vq3/vq3_enc_lib.c" -lm -o "${WORK}/vq3/libvq3_enc.so"
echo "== [5/6] 單元測試(kernel + 格式)=="
gcc -O2 -march=native -mavx512f -mf16c -lgomp \
-I"${WORK}/llama.cpp/ggml/include" -I"${WORK}/llama.cpp/ggml/src" \
"${HERE}/tests/test_vq3_port.c" \
"${WORK}/llama.cpp/build/ggml/src/libggml-cpu.a" \
"${WORK}/llama.cpp/build/ggml/src/libggml-base.a" \
"${WORK}/llama.cpp/build/ggml/src/libggml.a" \
-lm -lpthread -o "${WORK}/test_vq3_port" \
|| echo "note: 測試執行檔連結失敗可略過,不影響推論"
"${WORK}/test_vq3_port" 2>/dev/null || true
echo "== [6/6] 完成 =="
echo " 推論: ${BIN}/llama-cli -m <vq3.gguf> -p \"你好\" -n 256 -t ${T_DEC} -tb ${T_BATCH} -ngl 0"
echo " Server: ${BIN}/llama-server -m <vq3.gguf> -t ${T_DEC} -tb ${T_BATCH} -ngl 0 --host 0.0.0.0"
echo " 量化: python3 ${HERE}/scripts/build_ling_vq3.py --help"
if [[ "${DO_DOWNLOAD}" == "1" ]]; then
echo "== 下載 ${MODEL_NAME} =="
mkdir -p "${WORK}/models"
if command -v huggingface-cli >/dev/null 2>&1; then
huggingface-cli download "${HF_REPO}" "${MODEL_NAME}" --local-dir "${WORK}/models"
else
curl -L --fail -o "${WORK}/models/${MODEL_NAME}" "${MODEL_URL}"
fi
echo "== 推論冒煙測試 =="
"${BIN}/llama-cli" -m "${WORK}/models/${MODEL_NAME}" \
-p "你好,請用一句話介紹台灣。" -n 64 -t "${T_DEC}" -tb "${T_BATCH}" -ngl 0 --temp 0 -st
fi
if [[ "${DO_EXPERIMENTS}" == "1" ]]; then
MODEL="${WORK}/models/${MODEL_NAME}"
[[ -f "${MODEL}" ]] || { echo "需先 --download"; exit 1; }
echo "== PPL(wiki.test,256 chunks)=="
[[ -f "${WORK}/wiki.test.raw" ]] || { \
curl -sL -o "${WORK}/wt2.zip" "https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip" && \
unzip -o -j "${WORK}/wt2.zip" "wikitext-2-raw/wiki.test.raw" -d "${WORK}"; }
"${BIN}/llama-perplexity" -m "${MODEL}" -f "${WORK}/wiki.test.raw" \
-c 512 --chunks 256 -t "${T_BATCH}" -ngl 0
echo "== llama-bench(本機最佳執行緒)=="
"${BIN}/llama-bench" -t "${T_DEC},${T_BATCH}" -p 512 -n 128 -r 2 -m "${MODEL}" -ngl 0
fi
if [[ "${DO_CTX128K}" == "1" ]]; then
MODEL="${WORK}/models/${MODEL_NAME}"
[[ -f "${MODEL}" ]] || { echo "需先 --download"; exit 1; }
R="${WORK}/ctx128k"; mkdir -p "${R}"
echo "== 建立 ~120k token prompt =="
[[ -f "${R}/prompt_120k.txt" ]] || { \
curl -sL -o "${R}/wt2train.zip" "https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip" && \
unzip -o -j "${R}/wt2train.zip" "wikitext-2-raw/wiki.train.raw" -d "${R}" && \
head -c 520000 "${R}/wiki.train.raw" > "${R}/prompt_120k.txt"; }
echo "== 128k 上下文:prefill ~120k tokens + decode 64(-t ${T_DEC} -tb ${T_BATCH})=="
/usr/bin/time -v "${BIN}/llama-cli" -m "${MODEL}" -f "${R}/prompt_120k.txt" \
-c 131072 -n 64 -t "${T_DEC}" -tb "${T_BATCH}" -ub 1024 -ngl 0 --no-warmup --temp 0 -st \
2> >(tee "${R}/rss_128k.txt" >&2) | grep -E "prompt eval time|eval time|Maximum resident"
echo "(rss_128k.txt 內含尖峰駐留記憶體)"
fi