Download scripts/quantize2.py from nfa04/glm-ocr-onnx: direct link, hf CLI and curl.
- Browser
- Download file 4.03 kB
-
https://huggingface.co/nfa04/glm-ocr-onnx/resolve/main/scripts/quantize2.py
- Command line
-
hf download hf://nfa04/glm-ocr-onnx/scripts/quantize2.py
-
curl -L -o quantize2.py https://huggingface.co/nfa04/glm-ocr-onnx/resolve/main/scripts/quantize2.py
4.03 kB
| """Makes one quantised variant of one part: python3 quantize2.py vision|text SCHEME NAME -> ~/hw-bench/glm_onnx/parts/NAME/{part}.onnx(.data) | |
| SCHEME: nb:BITS:BLOCK[:asym][:keephead] weight-only MatMulNBits (BITS 4 or 8, symmetric unless asym; keephead leaves the output | |
| projection (lm_head) in 32 bits) | dyn8: dynamic int8 per channel (activations too) | fp16: weights stored as half floats | |
| Linear layers are turned from Gemm into MatMul first (the quantiser skips Gemm).""" | |
| import os, sys, shutil, numpy as np, onnx | |
| from onnx import helper, numpy_helper, TensorProto | |
| part, scheme, name = sys.argv[1:4] | |
| root = os.path.expanduser('~/hw-bench/glm_onnx'); dst = f'{root}/parts/{name}' | |
| shutil.rmtree(dst, ignore_errors=True); os.makedirs(dst) | |
| base = f'{root}/tmp/{part}_mm.onnx' | |
| if not os.path.exists(base): # the fp32 graph without the exporter's stale shape notes, linear layers as MatMul + Add | |
| os.makedirs(f'{root}/tmp', exist_ok=True) | |
| m = onnx.load(f'{root}/fp32/{part}.onnx'); del m.graph.value_info[:] | |
| inits = {i.name: i for i in m.graph.initializer}; new = [] | |
| for n in m.graph.node: | |
| if n.op_type == 'Gemm' and n.input[1] in inits: | |
| a = {x.name: helper.get_attribute_value(x) for x in n.attribute} | |
| w = numpy_helper.to_array(inits[n.input[1]]); w = w.T if a.get('transB', 0) else w | |
| wn = n.input[1] + '_t'; m.graph.initializer.append(numpy_helper.from_array(np.ascontiguousarray(w), wn)) | |
| mm = n.output[0] + '_mm' if len(n.input) > 2 else n.output[0] | |
| new.append(helper.make_node('MatMul', [n.input[0], wn], [mm], name=n.name + '_mm')) | |
| if len(n.input) > 2: new.append(helper.make_node('Add', [mm, n.input[2]], [n.output[0]], name=n.name + '_add')) | |
| else: new.append(n) | |
| del m.graph.node[:]; m.graph.node.extend(new) | |
| used = {i for n in m.graph.node for i in n.input}; keep = [i for i in m.graph.initializer if i.name in used] | |
| del m.graph.initializer[:]; m.graph.initializer.extend(keep) | |
| onnx.save_model(m, base, save_as_external_data=True, all_tensors_to_one_file=True, location=f'{part}_mm.onnx.data') | |
| m = onnx.load(base) | |
| kind = scheme.split(':')[0] | |
| if kind == 'nb': | |
| from onnxruntime.quantization.matmul_nbits_quantizer import MatMulNBitsQuantizer | |
| f = scheme.split(':'); bits, block = int(f[1]), int(f[2]); asym = 'asym' in f | |
| exclude = [] | |
| if 'keephead' in f: | |
| out = [n for n in m.graph.node if 'logits' in n.output[0] or n.output[0] == 'logits'] | |
| exclude = [n.name for n in out if n.op_type == 'MatMul'] | |
| print('keeping', exclude) | |
| attn = [] | |
| if 'mixed' in f: # attention weights (qkv, proj) in 8 bits, the rest in `bits`: first all the others, then the attention ones | |
| sizes = {i.name: tuple(i.dims) for i in m.graph.initializer} | |
| attn = [n.name for n in m.graph.node if n.op_type == 'MatMul' and sizes.get(n.input[1]) in ((1024, 3072), (1024, 1024))] | |
| print('attention matmuls:', len(attn)); exclude = attn | |
| q = MatMulNBitsQuantizer(m, block_size=block, is_symmetric=not asym, bits=bits, nodes_to_exclude=exclude) | |
| q.process(); m = q.model.model | |
| if attn: | |
| q8 = MatMulNBitsQuantizer(m, block_size=block, is_symmetric=not asym, bits=8); q8.process(); q = q8 | |
| q.model.save_model_to_file(f'{dst}/{part}.onnx', use_external_data_format=True) | |
| elif kind == 'dyn8': | |
| from onnxruntime.quantization import quantize_dynamic, QuantType | |
| quantize_dynamic(base, f'{dst}/{part}.onnx', weight_type=QuantType.QInt8, per_channel=True, op_types_to_quantize=['MatMul'], use_external_data_format=True) | |
| elif kind == 'fp16': | |
| from onnxruntime.transformers.float16 import convert_float_to_float16 | |
| m16 = convert_float_to_float16(m, keep_io_types=True, disable_shape_infer=True) | |
| onnx.save_model(m16, f'{dst}/{part}.onnx', save_as_external_data=True, all_tensors_to_one_file=True, location=f'{part}.onnx.data') | |
| print(name, part, [(fn, os.path.getsize(f'{dst}/{fn}') // 2**20) for fn in sorted(os.listdir(dst))]) | |