File size: 4,029 Bytes
15034cc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | """Makes one quantised variant of one part: python3 quantize2.py vision|text SCHEME NAME -> ~/hw-bench/glm_onnx/parts/NAME/{part}.onnx(.data)
SCHEME: nb:BITS:BLOCK[:asym][:keephead] weight-only MatMulNBits (BITS 4 or 8, symmetric unless asym; keephead leaves the output
projection (lm_head) in 32 bits) | dyn8: dynamic int8 per channel (activations too) | fp16: weights stored as half floats
Linear layers are turned from Gemm into MatMul first (the quantiser skips Gemm)."""
import os, sys, shutil, numpy as np, onnx
from onnx import helper, numpy_helper, TensorProto
part, scheme, name = sys.argv[1:4]
root = os.path.expanduser('~/hw-bench/glm_onnx'); dst = f'{root}/parts/{name}'
shutil.rmtree(dst, ignore_errors=True); os.makedirs(dst)
base = f'{root}/tmp/{part}_mm.onnx'
if not os.path.exists(base): # the fp32 graph without the exporter's stale shape notes, linear layers as MatMul + Add
os.makedirs(f'{root}/tmp', exist_ok=True)
m = onnx.load(f'{root}/fp32/{part}.onnx'); del m.graph.value_info[:]
inits = {i.name: i for i in m.graph.initializer}; new = []
for n in m.graph.node:
if n.op_type == 'Gemm' and n.input[1] in inits:
a = {x.name: helper.get_attribute_value(x) for x in n.attribute}
w = numpy_helper.to_array(inits[n.input[1]]); w = w.T if a.get('transB', 0) else w
wn = n.input[1] + '_t'; m.graph.initializer.append(numpy_helper.from_array(np.ascontiguousarray(w), wn))
mm = n.output[0] + '_mm' if len(n.input) > 2 else n.output[0]
new.append(helper.make_node('MatMul', [n.input[0], wn], [mm], name=n.name + '_mm'))
if len(n.input) > 2: new.append(helper.make_node('Add', [mm, n.input[2]], [n.output[0]], name=n.name + '_add'))
else: new.append(n)
del m.graph.node[:]; m.graph.node.extend(new)
used = {i for n in m.graph.node for i in n.input}; keep = [i for i in m.graph.initializer if i.name in used]
del m.graph.initializer[:]; m.graph.initializer.extend(keep)
onnx.save_model(m, base, save_as_external_data=True, all_tensors_to_one_file=True, location=f'{part}_mm.onnx.data')
m = onnx.load(base)
kind = scheme.split(':')[0]
if kind == 'nb':
from onnxruntime.quantization.matmul_nbits_quantizer import MatMulNBitsQuantizer
f = scheme.split(':'); bits, block = int(f[1]), int(f[2]); asym = 'asym' in f
exclude = []
if 'keephead' in f:
out = [n for n in m.graph.node if 'logits' in n.output[0] or n.output[0] == 'logits']
exclude = [n.name for n in out if n.op_type == 'MatMul']
print('keeping', exclude)
attn = []
if 'mixed' in f: # attention weights (qkv, proj) in 8 bits, the rest in `bits`: first all the others, then the attention ones
sizes = {i.name: tuple(i.dims) for i in m.graph.initializer}
attn = [n.name for n in m.graph.node if n.op_type == 'MatMul' and sizes.get(n.input[1]) in ((1024, 3072), (1024, 1024))]
print('attention matmuls:', len(attn)); exclude = attn
q = MatMulNBitsQuantizer(m, block_size=block, is_symmetric=not asym, bits=bits, nodes_to_exclude=exclude)
q.process(); m = q.model.model
if attn:
q8 = MatMulNBitsQuantizer(m, block_size=block, is_symmetric=not asym, bits=8); q8.process(); q = q8
q.model.save_model_to_file(f'{dst}/{part}.onnx', use_external_data_format=True)
elif kind == 'dyn8':
from onnxruntime.quantization import quantize_dynamic, QuantType
quantize_dynamic(base, f'{dst}/{part}.onnx', weight_type=QuantType.QInt8, per_channel=True, op_types_to_quantize=['MatMul'], use_external_data_format=True)
elif kind == 'fp16':
from onnxruntime.transformers.float16 import convert_float_to_float16
m16 = convert_float_to_float16(m, keep_io_types=True, disable_shape_infer=True)
onnx.save_model(m16, f'{dst}/{part}.onnx', save_as_external_data=True, all_tensors_to_one_file=True, location=f'{part}.onnx.data')
print(name, part, [(fn, os.path.getsize(f'{dst}/{fn}') // 2**20) for fn in sorted(os.listdir(dst))])
|