lingtiny30vq3 / scripts /build_ling_vq3.py
HelloSun's picture
LUT-VQ3-64: machine-tuned (threads/NUMA/ubatch), full 128k ctx data, one-click build, results
511ba50 verified
Raw History Blame Contribute Delete
10.5 kB
#!/usr/bin/env python3
"""
build_ling_vq3.py — Build the inclusionAI Ling-3.0-tiny LUT-VQ3-64 hybrid GGUF.
Based on llama-modchip's scripts/build_nemotron_vq3.py, adapted for the
BailingMoeV3 (bailing_hybrid) architecture and for the VQ3 port to upstream
llama.cpp @ 61dbf492 (GGML_TYPE_VQ3_0 = 44 in the port, 24 B / 64 W, 3.00 bpw).
Recipes:
hybrid (default, domain-aware — mirrors the modchip VQ3-64 layout):
blk.*.ffn_gate_exps.weight + blk.*.ffn_up_exps.weight -> LUT-VQ3-64 (3.00 bpw)
blk.*.ffn_down_exps.weight -> Q6_K (6.56 bpw)
attention / shared experts / dense ffn / embeddings -> Q8_0 (8.50 bpw)
max (--down-vq3):
additionally blk.*.ffn_down_exps.weight -> LUT-VQ3-64 (3.00 bpw)
The routed-expert tensors are encoded directly from the uncompressed BF16 master
GGUF (inclusionAI/Ling-3.0-tiny-bf16.gguf) with the modchip's OpenMP AVX-512
encoder (vq3/vq3_enc_lib.c, 128-entry 3D Lloyd-Max codebook).
Usage:
python3 build_ling_vq3.py --base-gguf models/Ling-3.0-tiny-base.gguf \
--bf16-gguf models/Ling-3.0-tiny-bf16.gguf \
--output models/Ling-3.0-tiny-VQ3.gguf
"""
import argparse
import ctypes
import mmap
import os
import struct
import subprocess
import sys
import time
from pathlib import Path
import numpy as np
GGML_TYPE_BF16 = 30
GGML_TYPE_VQ3_0 = 44 # llama-modchip port: appended after upstream GGML_TYPE_F8_E4M3 = 43
def load_vq3_encoder(lib_path: str = "") -> ctypes.CDLL:
here = Path(__file__).resolve().parent
# encoder source lives next to this script (HF repo layout) or in a llama-modchip checkout
so_candidates = [
Path(lib_path) if lib_path else None,
here.parent / "vq3" / "libvq3_enc.so",
here.parent.parent / "llama-modchip" / "vq3" / "libvq3_enc.so",
]
so_path = next((c for c in so_candidates if c and c.is_file()), None)
src_candidates = [
here.parent / "vq3" / "vq3_enc_lib.c",
here.parent.parent / "llama-modchip" / "vq3" / "vq3_enc_lib.c",
]
c_src = next((c for c in src_candidates if c.is_file()), None)
if so_path is None:
if not c_src:
raise FileNotFoundError(f"Missing encoder library and source {src_candidates}")
so_path = c_src.with_suffix(".so").parent / "libvq3_enc.so"
print(f"[*] Compiling {c_src} -> {so_path} ...", flush=True)
subprocess.check_call([
"gcc", "-O3", "-march=native", "-mavx512f", "-mf16c",
"-fopenmp", "-shared", "-fPIC", str(c_src), "-lm", "-o", str(so_path),
])
lib = ctypes.CDLL(str(so_path))
lib.quantize_bf16_to_vq3_0_omp.argtypes = [
ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_int64,
]
lib.quantize_bf16_to_vq3_0_omp.restype = None
return lib
def parse_gguf(f):
"""Minimal GGUF v2/v3 reader: returns metadata KV blob + tensor directory."""
f.seek(0)
magic, version, n_tensors, n_kv = struct.unpack("<4sIQQ", f.read(24))
if magic != b"GGUF":
raise ValueError(f"Not a GGUF file (magic={magic!r})")
def read_str():
length = struct.unpack("<Q", f.read(8))[0]
return f.read(length).decode("utf-8", errors="replace")
def skip_val(vtype):
if vtype in (0, 1, 7):
f.seek(1, 1)
elif vtype in (2, 3):
f.seek(2, 1)
elif vtype in (4, 5, 6):
f.seek(4, 1)
elif vtype in (10, 11, 12):
f.seek(8, 1)
elif vtype == 8:
length = struct.unpack("<Q", f.read(8))[0]
f.seek(length, 1)
elif vtype == 9:
atype, alen = struct.unpack("<IQ", f.read(12))
for _ in range(alen):
skip_val(atype)
else:
raise ValueError(f"Unknown GGUF metadata type {vtype}")
for _ in range(n_kv):
_ = read_str()
vtype = struct.unpack("<I", f.read(4))[0]
skip_val(vtype)
kv_end = f.tell()
tensors = []
for _ in range(n_tensors):
name = read_str()
n_dims = struct.unpack("<I", f.read(4))[0]
dims = [struct.unpack("<Q", f.read(8))[0] for _ in range(n_dims)]
ttype, offset = struct.unpack("<IQ", f.read(12))
tensors.append({
"name": name,
"n_dims": n_dims,
"dims": dims,
"type": ttype,
"offset": offset,
})
header_end = f.tell()
alignment = 32
data_start = (header_end + alignment - 1) // alignment * alignment
return version, n_tensors, n_kv, kv_end, tensors, header_end, data_start
def main() -> int:
ap = argparse.ArgumentParser(description="Build Ling-3.0-tiny LUT-VQ3-64 hybrid GGUF")
ap.add_argument("--base-gguf", required=True,
help="Quantized base GGUF (e.g. Q8_0 with --tensor-type ffn_down_exps=Q6_K)")
ap.add_argument("--bf16-gguf", required=True,
help="Uncompressed BF16 master GGUF (VQ3 encode source)")
ap.add_argument("--output", required=True, help="Output VQ3 hybrid GGUF path")
ap.add_argument("--down-vq3", action="store_true",
help="Also encode ffn_down_exps to VQ3-64 (aggressive 'max' recipe)")
ap.add_argument("--lib", default=os.environ.get("VQ3_ENC_LIB", ""),
help="Optional path to libvq3_enc.so")
args = ap.parse_args()
lib = load_vq3_encoder(args.lib)
vq3_match = ("ffn_gate_exps", "ffn_up_exps") + (("ffn_down_exps",) if args.down_vq3 else ())
t0 = time.time()
with open(args.base_gguf, "rb") as ff, open(args.bf16_gguf, "rb") as fb:
_, _, _, kv_end, f_tensors, f_hdr_end, f_data_start = parse_gguf(ff)
_, _, _, _, b_tensors, _, b_data_start = parse_gguf(fb)
bf16_map = {t["name"]: t for t in b_tensors}
sorted_ft = sorted(f_tensors, key=lambda x: x["offset"])
ff.seek(0, os.SEEK_END)
f_file_size = ff.tell()
new_offset = 0
alignment = 32
vq3_target_total = 0
for t in f_tensors:
n_el = 1
for d in t["dims"]:
n_el *= d
t["n_el"] = n_el
if any(m in t["name"] for m in vq3_match) and t["name"].endswith("exps.weight"):
if n_el % 64:
raise ValueError(f"{t['name']}: {n_el} not divisible by 64")
t["new_type"] = GGML_TYPE_VQ3_0
t["new_bytes"] = (n_el // 64) * 24
t["old_bytes"] = (n_el // 64) * 30 if t["type"] == GGML_TYPE_BF16 else None
if t["old_bytes"] is None:
t["old_bytes"] = next(
(sorted_ft[i + 1]["offset"] - t["offset"] for i in range(len(sorted_ft) - 1)
if sorted_ft[i] is t), f_file_size - (f_data_start + t["offset"]))
vq3_target_total += 1
else:
t["new_type"] = t["type"]
idx_s = sorted_ft.index(t)
if idx_s + 1 < len(sorted_ft):
span = sorted_ft[idx_s + 1]["offset"] - t["offset"]
else:
span = f_file_size - (f_data_start + t["offset"])
t["old_bytes"] = span
t["new_bytes"] = span
t["new_offset"] = new_offset
padded = (t["new_bytes"] + alignment - 1) // alignment * alignment
new_offset += padded
out_gb = (f_data_start + new_offset) / (1024 ** 3)
print(f"Parsed headers: {len(f_tensors)} tensors, encoding {vq3_target_total} expert tensors "
f"to VQ3_0(44), output size ~{out_gb:.2f} GiB", flush=True)
mm_f = mmap.mmap(ff.fileno(), 0, access=mmap.ACCESS_READ)
mm_b = mmap.mmap(fb.fileno(), 0, access=mmap.ACCESS_READ)
with open(args.output, "wb") as fout:
# metadata KV (verbatim from base) + rewritten tensor directory
ff.seek(0)
fout.write(ff.read(kv_end))
for t in f_tensors:
nb = t["name"].encode("utf-8")
fout.write(struct.pack("<Q", len(nb)))
fout.write(nb)
fout.write(struct.pack("<I", t["n_dims"]))
for d in t["dims"]:
fout.write(struct.pack("<Q", d))
fout.write(struct.pack("<IQ", t["new_type"], t["new_offset"]))
assert fout.tell() == f_hdr_end, (fout.tell(), f_hdr_end)
if f_data_start > f_hdr_end:
fout.write(b"\x00" * (f_data_start - f_hdr_end))
vq3_cnt = 0
for t in f_tensors:
target_pos = f_data_start + t["new_offset"]
cur_pos = fout.tell()
if target_pos > cur_pos:
fout.write(b"\x00" * (target_pos - cur_pos))
assert fout.tell() == target_pos
if t["new_type"] == GGML_TYPE_VQ3_0:
bt = bf16_map[t["name"]]
t_type = bt["type"]
assert t_type == GGML_TYPE_BF16, \
f"Expected BF16 ({GGML_TYPE_BF16}) for {t['name']}, got {t_type}"
n_el = t["n_el"]
ncol = t["dims"][0]
nrow = n_el // ncol
b_off = b_data_start + bt["offset"]
src_ptr = np.frombuffer(mm_b, dtype=np.uint8, count=n_el * 2,
offset=b_off).ctypes.data
out_buf = np.empty(t["new_bytes"], dtype=np.uint8)
dst_ptr = out_buf.ctypes.data
ts = time.time()
lib.quantize_bf16_to_vq3_0_omp(src_ptr, dst_ptr, nrow, ncol)
fout.write(out_buf.tobytes())
vq3_cnt += 1
print(f"[{vq3_cnt:2d}/{vq3_target_total}] {t['name']} "
f"({n_el / 1e6:.1f}M w) BF16 -> VQ3_0 in {time.time() - ts:.2f}s",
flush=True)
else:
f_off = f_data_start + t["offset"]
fout.write(mm_f[f_off:f_off + t["old_bytes"]])
rem = fout.tell() % alignment
if rem:
fout.write(b"\x00" * (alignment - rem))
mm_f.close()
mm_b.close()
print(f"DONE in {time.time() - t0:.1f}s: {args.output} "
f"({os.path.getsize(args.output) / (1024 ** 3):.2f} GiB)", flush=True)
return 0
if __name__ == "__main__":
raise SystemExit(main())