glm-ocr-onnx / scripts /quantize2.py
nfa04's picture
ONNX files (8-bit vision, 4-bit language model, int8 embeddings) and the scripts that made them
15034cc verified
Raw History Blame Contribute Delete
4.03 kB
"""Makes one quantised variant of one part: python3 quantize2.py vision|text SCHEME NAME -> ~/hw-bench/glm_onnx/parts/NAME/{part}.onnx(.data)
SCHEME: nb:BITS:BLOCK[:asym][:keephead] weight-only MatMulNBits (BITS 4 or 8, symmetric unless asym; keephead leaves the output
projection (lm_head) in 32 bits) | dyn8: dynamic int8 per channel (activations too) | fp16: weights stored as half floats
Linear layers are turned from Gemm into MatMul first (the quantiser skips Gemm)."""
import os, sys, shutil, numpy as np, onnx
from onnx import helper, numpy_helper, TensorProto
part, scheme, name = sys.argv[1:4]
root = os.path.expanduser('~/hw-bench/glm_onnx'); dst = f'{root}/parts/{name}'
shutil.rmtree(dst, ignore_errors=True); os.makedirs(dst)
base = f'{root}/tmp/{part}_mm.onnx'
if not os.path.exists(base): # the fp32 graph without the exporter's stale shape notes, linear layers as MatMul + Add
os.makedirs(f'{root}/tmp', exist_ok=True)
m = onnx.load(f'{root}/fp32/{part}.onnx'); del m.graph.value_info[:]
inits = {i.name: i for i in m.graph.initializer}; new = []
for n in m.graph.node:
if n.op_type == 'Gemm' and n.input[1] in inits:
a = {x.name: helper.get_attribute_value(x) for x in n.attribute}
w = numpy_helper.to_array(inits[n.input[1]]); w = w.T if a.get('transB', 0) else w
wn = n.input[1] + '_t'; m.graph.initializer.append(numpy_helper.from_array(np.ascontiguousarray(w), wn))
mm = n.output[0] + '_mm' if len(n.input) > 2 else n.output[0]
new.append(helper.make_node('MatMul', [n.input[0], wn], [mm], name=n.name + '_mm'))
if len(n.input) > 2: new.append(helper.make_node('Add', [mm, n.input[2]], [n.output[0]], name=n.name + '_add'))
else: new.append(n)
del m.graph.node[:]; m.graph.node.extend(new)
used = {i for n in m.graph.node for i in n.input}; keep = [i for i in m.graph.initializer if i.name in used]
del m.graph.initializer[:]; m.graph.initializer.extend(keep)
onnx.save_model(m, base, save_as_external_data=True, all_tensors_to_one_file=True, location=f'{part}_mm.onnx.data')
m = onnx.load(base)
kind = scheme.split(':')[0]
if kind == 'nb':
from onnxruntime.quantization.matmul_nbits_quantizer import MatMulNBitsQuantizer
f = scheme.split(':'); bits, block = int(f[1]), int(f[2]); asym = 'asym' in f
exclude = []
if 'keephead' in f:
out = [n for n in m.graph.node if 'logits' in n.output[0] or n.output[0] == 'logits']
exclude = [n.name for n in out if n.op_type == 'MatMul']
print('keeping', exclude)
attn = []
if 'mixed' in f: # attention weights (qkv, proj) in 8 bits, the rest in `bits`: first all the others, then the attention ones
sizes = {i.name: tuple(i.dims) for i in m.graph.initializer}
attn = [n.name for n in m.graph.node if n.op_type == 'MatMul' and sizes.get(n.input[1]) in ((1024, 3072), (1024, 1024))]
print('attention matmuls:', len(attn)); exclude = attn
q = MatMulNBitsQuantizer(m, block_size=block, is_symmetric=not asym, bits=bits, nodes_to_exclude=exclude)
q.process(); m = q.model.model
if attn:
q8 = MatMulNBitsQuantizer(m, block_size=block, is_symmetric=not asym, bits=8); q8.process(); q = q8
q.model.save_model_to_file(f'{dst}/{part}.onnx', use_external_data_format=True)
elif kind == 'dyn8':
from onnxruntime.quantization import quantize_dynamic, QuantType
quantize_dynamic(base, f'{dst}/{part}.onnx', weight_type=QuantType.QInt8, per_channel=True, op_types_to_quantize=['MatMul'], use_external_data_format=True)
elif kind == 'fp16':
from onnxruntime.transformers.float16 import convert_float_to_float16
m16 = convert_float_to_float16(m, keep_io_types=True, disable_shape_infer=True)
onnx.save_model(m16, f'{dst}/{part}.onnx', save_as_external_data=True, all_tensors_to_one_file=True, location=f'{part}.onnx.data')
print(name, part, [(fn, os.path.getsize(f'{dst}/{fn}') // 2**20) for fn in sorted(os.listdir(dst))])