#!/usr/bin/env python3 """ build_ling_vq3.py — Build the inclusionAI Ling-3.0-tiny LUT-VQ3-64 hybrid GGUF. Based on llama-modchip's scripts/build_nemotron_vq3.py, adapted for the BailingMoeV3 (bailing_hybrid) architecture and for the VQ3 port to upstream llama.cpp @ 61dbf492 (GGML_TYPE_VQ3_0 = 44 in the port, 24 B / 64 W, 3.00 bpw). Recipes: hybrid (default, domain-aware — mirrors the modchip VQ3-64 layout): blk.*.ffn_gate_exps.weight + blk.*.ffn_up_exps.weight -> LUT-VQ3-64 (3.00 bpw) blk.*.ffn_down_exps.weight -> Q6_K (6.56 bpw) attention / shared experts / dense ffn / embeddings -> Q8_0 (8.50 bpw) max (--down-vq3): additionally blk.*.ffn_down_exps.weight -> LUT-VQ3-64 (3.00 bpw) The routed-expert tensors are encoded directly from the uncompressed BF16 master GGUF (inclusionAI/Ling-3.0-tiny-bf16.gguf) with the modchip's OpenMP AVX-512 encoder (vq3/vq3_enc_lib.c, 128-entry 3D Lloyd-Max codebook). Usage: python3 build_ling_vq3.py --base-gguf models/Ling-3.0-tiny-base.gguf \ --bf16-gguf models/Ling-3.0-tiny-bf16.gguf \ --output models/Ling-3.0-tiny-VQ3.gguf """ import argparse import ctypes import mmap import os import struct import subprocess import sys import time from pathlib import Path import numpy as np GGML_TYPE_BF16 = 30 GGML_TYPE_VQ3_0 = 44 # llama-modchip port: appended after upstream GGML_TYPE_F8_E4M3 = 43 def load_vq3_encoder(lib_path: str = "") -> ctypes.CDLL: here = Path(__file__).resolve().parent # encoder source lives next to this script (HF repo layout) or in a llama-modchip checkout so_candidates = [ Path(lib_path) if lib_path else None, here.parent / "vq3" / "libvq3_enc.so", here.parent.parent / "llama-modchip" / "vq3" / "libvq3_enc.so", ] so_path = next((c for c in so_candidates if c and c.is_file()), None) src_candidates = [ here.parent / "vq3" / "vq3_enc_lib.c", here.parent.parent / "llama-modchip" / "vq3" / "vq3_enc_lib.c", ] c_src = next((c for c in src_candidates if c.is_file()), None) if so_path is None: if not c_src: raise FileNotFoundError(f"Missing encoder library and source {src_candidates}") so_path = c_src.with_suffix(".so").parent / "libvq3_enc.so" print(f"[*] Compiling {c_src} -> {so_path} ...", flush=True) subprocess.check_call([ "gcc", "-O3", "-march=native", "-mavx512f", "-mf16c", "-fopenmp", "-shared", "-fPIC", str(c_src), "-lm", "-o", str(so_path), ]) lib = ctypes.CDLL(str(so_path)) lib.quantize_bf16_to_vq3_0_omp.argtypes = [ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64, ctypes.c_int64, ] lib.quantize_bf16_to_vq3_0_omp.restype = None return lib def parse_gguf(f): """Minimal GGUF v2/v3 reader: returns metadata KV blob + tensor directory.""" f.seek(0) magic, version, n_tensors, n_kv = struct.unpack("<4sIQQ", f.read(24)) if magic != b"GGUF": raise ValueError(f"Not a GGUF file (magic={magic!r})") def read_str(): length = struct.unpack(" int: ap = argparse.ArgumentParser(description="Build Ling-3.0-tiny LUT-VQ3-64 hybrid GGUF") ap.add_argument("--base-gguf", required=True, help="Quantized base GGUF (e.g. Q8_0 with --tensor-type ffn_down_exps=Q6_K)") ap.add_argument("--bf16-gguf", required=True, help="Uncompressed BF16 master GGUF (VQ3 encode source)") ap.add_argument("--output", required=True, help="Output VQ3 hybrid GGUF path") ap.add_argument("--down-vq3", action="store_true", help="Also encode ffn_down_exps to VQ3-64 (aggressive 'max' recipe)") ap.add_argument("--lib", default=os.environ.get("VQ3_ENC_LIB", ""), help="Optional path to libvq3_enc.so") args = ap.parse_args() lib = load_vq3_encoder(args.lib) vq3_match = ("ffn_gate_exps", "ffn_up_exps") + (("ffn_down_exps",) if args.down_vq3 else ()) t0 = time.time() with open(args.base_gguf, "rb") as ff, open(args.bf16_gguf, "rb") as fb: _, _, _, kv_end, f_tensors, f_hdr_end, f_data_start = parse_gguf(ff) _, _, _, _, b_tensors, _, b_data_start = parse_gguf(fb) bf16_map = {t["name"]: t for t in b_tensors} sorted_ft = sorted(f_tensors, key=lambda x: x["offset"]) ff.seek(0, os.SEEK_END) f_file_size = ff.tell() new_offset = 0 alignment = 32 vq3_target_total = 0 for t in f_tensors: n_el = 1 for d in t["dims"]: n_el *= d t["n_el"] = n_el if any(m in t["name"] for m in vq3_match) and t["name"].endswith("exps.weight"): if n_el % 64: raise ValueError(f"{t['name']}: {n_el} not divisible by 64") t["new_type"] = GGML_TYPE_VQ3_0 t["new_bytes"] = (n_el // 64) * 24 t["old_bytes"] = (n_el // 64) * 30 if t["type"] == GGML_TYPE_BF16 else None if t["old_bytes"] is None: t["old_bytes"] = next( (sorted_ft[i + 1]["offset"] - t["offset"] for i in range(len(sorted_ft) - 1) if sorted_ft[i] is t), f_file_size - (f_data_start + t["offset"])) vq3_target_total += 1 else: t["new_type"] = t["type"] idx_s = sorted_ft.index(t) if idx_s + 1 < len(sorted_ft): span = sorted_ft[idx_s + 1]["offset"] - t["offset"] else: span = f_file_size - (f_data_start + t["offset"]) t["old_bytes"] = span t["new_bytes"] = span t["new_offset"] = new_offset padded = (t["new_bytes"] + alignment - 1) // alignment * alignment new_offset += padded out_gb = (f_data_start + new_offset) / (1024 ** 3) print(f"Parsed headers: {len(f_tensors)} tensors, encoding {vq3_target_total} expert tensors " f"to VQ3_0(44), output size ~{out_gb:.2f} GiB", flush=True) mm_f = mmap.mmap(ff.fileno(), 0, access=mmap.ACCESS_READ) mm_b = mmap.mmap(fb.fileno(), 0, access=mmap.ACCESS_READ) with open(args.output, "wb") as fout: # metadata KV (verbatim from base) + rewritten tensor directory ff.seek(0) fout.write(ff.read(kv_end)) for t in f_tensors: nb = t["name"].encode("utf-8") fout.write(struct.pack(" f_hdr_end: fout.write(b"\x00" * (f_data_start - f_hdr_end)) vq3_cnt = 0 for t in f_tensors: target_pos = f_data_start + t["new_offset"] cur_pos = fout.tell() if target_pos > cur_pos: fout.write(b"\x00" * (target_pos - cur_pos)) assert fout.tell() == target_pos if t["new_type"] == GGML_TYPE_VQ3_0: bt = bf16_map[t["name"]] t_type = bt["type"] assert t_type == GGML_TYPE_BF16, \ f"Expected BF16 ({GGML_TYPE_BF16}) for {t['name']}, got {t_type}" n_el = t["n_el"] ncol = t["dims"][0] nrow = n_el // ncol b_off = b_data_start + bt["offset"] src_ptr = np.frombuffer(mm_b, dtype=np.uint8, count=n_el * 2, offset=b_off).ctypes.data out_buf = np.empty(t["new_bytes"], dtype=np.uint8) dst_ptr = out_buf.ctypes.data ts = time.time() lib.quantize_bf16_to_vq3_0_omp(src_ptr, dst_ptr, nrow, ncol) fout.write(out_buf.tobytes()) vq3_cnt += 1 print(f"[{vq3_cnt:2d}/{vq3_target_total}] {t['name']} " f"({n_el / 1e6:.1f}M w) BF16 -> VQ3_0 in {time.time() - ts:.2f}s", flush=True) else: f_off = f_data_start + t["offset"] fout.write(mm_f[f_off:f_off + t["old_bytes"]]) rem = fout.tell() % alignment if rem: fout.write(b"\x00" * (alignment - rem)) mm_f.close() mm_b.close() print(f"DONE in {time.time() - t0:.1f}s: {args.output} " f"({os.path.getsize(args.output) / (1024 ** 3):.2f} GiB)", flush=True) return 0 if __name__ == "__main__": raise SystemExit(main())