#!/usr/bin/env bash # ============================================================================= # build_and_run.sh — LUT-VQ3-64 for Ling-3.0-tiny:一鍵編譯 + 驗證 + 下載 + 實驗 # # bash build_and_run.sh # 編譯 + 單元測試 # bash build_and_run.sh --download # 另外下載 VQ3 GGUF 並做推論冒煙測試 # bash build_and_run.sh --experiments # 另外跑 PPL + llama-bench + 生成樣本 # bash build_and_run.sh --ctx128k # 另外跑 128k 上下文實驗(prefill/decode/RSS) # bash build_and_run.sh --all # 全部 # # 需求:git, cmake>=3.14, gcc/g++(x86-64 建議 AVX-512;VQ3 LUT 核心需 VBMI), # python3(僅重量化時需要) # 模型:inclusionAI/Ling-3.0-tiny(MIT)之 LUT-VQ3-64 量化(專家張量 3.00 bpw) # # ※ 本機最佳化(2×40C Xeon,雙 NUMA)實測結論: # - decode(tg)用 -t 32 最快(38.9 t/s);prefill(pp)用 -t 64 最快(353+ t/s) # - 故預設採用 -t 32 -tb 64(decode 32 執行緒 / prefill 64 執行緒) # - 【不要】加 --numa distribute:實測 tg@t80 崩至 0.9 t/s、pp@t48 −21% # - 80 執行緒會跨 NUMA 抖動(pp 崩至 ~70 t/s),避免使用 # ============================================================================= set -euo pipefail LLAMA_REPO="https://github.com/ggml-org/llama.cpp" LLAMA_COMMIT="61dbf49228c8a96b72e40c89d93cb7c003411f8c" HF_REPO="HelloSun/lingtiny30vq3" MODEL_NAME="Ling-3.0-tiny-VQ3.gguf" MODEL_URL="https://huggingface.co/${HF_REPO}/resolve/main/${MODEL_NAME}" HERE="$(cd "$(dirname "$0")" && pwd)" WORK="${HERE}/work" JOBS="$(nproc)" T_DEC=32 # decode 最佳執行緒數(本機實測) T_BATCH=64 # prefill/batch 最佳執行緒數(本機實測) DO_DOWNLOAD=0; DO_EXPERIMENTS=0; DO_CTX128K=0 while [[ $# -gt 0 ]]; do case "$1" in --jobs) JOBS="$2"; shift 2;; --download) DO_DOWNLOAD=1; shift;; --experiments) DO_EXPERIMENTS=1; shift;; --ctx128k) DO_CTX128K=1; shift;; --all) DO_DOWNLOAD=1; DO_EXPERIMENTS=1; DO_CTX128K=1; shift;; *) echo "unknown arg: $1"; exit 1;; esac done echo "== [1/6] 取得 llama.cpp @ ${LLAMA_COMMIT:0:9} ==" mkdir -p "${WORK}" if [[ ! -d "${WORK}/llama.cpp" ]]; then git clone "${LLAMA_REPO}" "${WORK}/llama.cpp" fi git -C "${WORK}/llama.cpp" fetch --depth 1 origin "${LLAMA_COMMIT}" 2>/dev/null || true git -C "${WORK}/llama.cpp" checkout -q --detach "${LLAMA_COMMIT}" echo "== [2/6] 套用 LUT-VQ3-64 補丁 ==" if git -C "${WORK}/llama.cpp" apply --check "${HERE}/patches/0001-lut-vq3-64-avx512-vbmi-port-to-61dbf492.patch" 2>/dev/null; then git -C "${WORK}/llama.cpp" apply "${HERE}/patches/0001-lut-vq3-64-avx512-vbmi-port-to-61dbf492.patch" echo "patch applied" elif grep -rq "GGML_TYPE_VQ3_0" "${WORK}/llama.cpp/ggml/include/ggml.h"; then echo "patch already applied" else echo "ERROR: patch does not apply"; exit 1 fi echo "== [3/6] 編譯(-j ${JOBS},GGML_NATIVE=ON:AVX-512/AMX 自動啟用)==" cmake -S "${WORK}/llama.cpp" -B "${WORK}/llama.cpp/build" \ -DGGML_NATIVE=ON -DLLAMA_CURL=OFF -DBUILD_SHARED_LIBS=OFF \ -DCMAKE_BUILD_TYPE=Release cmake --build "${WORK}/llama.cpp/build" -j "${JOBS}" \ --target llama-cli llama-server llama-quantize llama-perplexity llama-bench BIN="${WORK}/llama.cpp/build/bin" echo "== [4/6] 編譯 VQ3 編碼器函式庫(量化用)==" mkdir -p "${WORK}/vq3" cp "${HERE}/vq3/vq3_enc_lib.c" "${HERE}/vq3/vq3_codebook.h" "${WORK}/vq3/" gcc -O3 -march=native -mavx512f -mf16c -fopenmp -shared -fPIC \ "${WORK}/vq3/vq3_enc_lib.c" -lm -o "${WORK}/vq3/libvq3_enc.so" echo "== [5/6] 單元測試(kernel + 格式)==" gcc -O2 -march=native -mavx512f -mf16c -lgomp \ -I"${WORK}/llama.cpp/ggml/include" -I"${WORK}/llama.cpp/ggml/src" \ "${HERE}/tests/test_vq3_port.c" \ "${WORK}/llama.cpp/build/ggml/src/libggml-cpu.a" \ "${WORK}/llama.cpp/build/ggml/src/libggml-base.a" \ "${WORK}/llama.cpp/build/ggml/src/libggml.a" \ -lm -lpthread -o "${WORK}/test_vq3_port" \ || echo "note: 測試執行檔連結失敗可略過,不影響推論" "${WORK}/test_vq3_port" 2>/dev/null || true echo "== [6/6] 完成 ==" echo " 推論: ${BIN}/llama-cli -m -p \"你好\" -n 256 -t ${T_DEC} -tb ${T_BATCH} -ngl 0" echo " Server: ${BIN}/llama-server -m -t ${T_DEC} -tb ${T_BATCH} -ngl 0 --host 0.0.0.0" echo " 量化: python3 ${HERE}/scripts/build_ling_vq3.py --help" if [[ "${DO_DOWNLOAD}" == "1" ]]; then echo "== 下載 ${MODEL_NAME} ==" mkdir -p "${WORK}/models" if command -v huggingface-cli >/dev/null 2>&1; then huggingface-cli download "${HF_REPO}" "${MODEL_NAME}" --local-dir "${WORK}/models" else curl -L --fail -o "${WORK}/models/${MODEL_NAME}" "${MODEL_URL}" fi echo "== 推論冒煙測試 ==" "${BIN}/llama-cli" -m "${WORK}/models/${MODEL_NAME}" \ -p "你好,請用一句話介紹台灣。" -n 64 -t "${T_DEC}" -tb "${T_BATCH}" -ngl 0 --temp 0 -st fi if [[ "${DO_EXPERIMENTS}" == "1" ]]; then MODEL="${WORK}/models/${MODEL_NAME}" [[ -f "${MODEL}" ]] || { echo "需先 --download"; exit 1; } echo "== PPL(wiki.test,256 chunks)==" [[ -f "${WORK}/wiki.test.raw" ]] || { \ curl -sL -o "${WORK}/wt2.zip" "https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip" && \ unzip -o -j "${WORK}/wt2.zip" "wikitext-2-raw/wiki.test.raw" -d "${WORK}"; } "${BIN}/llama-perplexity" -m "${MODEL}" -f "${WORK}/wiki.test.raw" \ -c 512 --chunks 256 -t "${T_BATCH}" -ngl 0 echo "== llama-bench(本機最佳執行緒)==" "${BIN}/llama-bench" -t "${T_DEC},${T_BATCH}" -p 512 -n 128 -r 2 -m "${MODEL}" -ngl 0 fi if [[ "${DO_CTX128K}" == "1" ]]; then MODEL="${WORK}/models/${MODEL_NAME}" [[ -f "${MODEL}" ]] || { echo "需先 --download"; exit 1; } R="${WORK}/ctx128k"; mkdir -p "${R}" echo "== 建立 ~120k token prompt ==" [[ -f "${R}/prompt_120k.txt" ]] || { \ curl -sL -o "${R}/wt2train.zip" "https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip" && \ unzip -o -j "${R}/wt2train.zip" "wikitext-2-raw/wiki.train.raw" -d "${R}" && \ head -c 520000 "${R}/wiki.train.raw" > "${R}/prompt_120k.txt"; } echo "== 128k 上下文:prefill ~120k tokens + decode 64(-t ${T_DEC} -tb ${T_BATCH})==" /usr/bin/time -v "${BIN}/llama-cli" -m "${MODEL}" -f "${R}/prompt_120k.txt" \ -c 131072 -n 64 -t "${T_DEC}" -tb "${T_BATCH}" -ub 1024 -ngl 0 --no-warmup --temp 0 -st \ 2> >(tee "${R}/rss_128k.txt" >&2) | grep -E "prompt eval time|eval time|Maximum resident" echo "(rss_128k.txt 內含尖峰駐留記憶體)" fi