File size: 4,029 Bytes
15034cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
"""Makes one quantised variant of one part:  python3 quantize2.py vision|text SCHEME NAME   ->  ~/hw-bench/glm_onnx/parts/NAME/{part}.onnx(.data)
SCHEME: nb:BITS:BLOCK[:asym][:keephead]  weight-only MatMulNBits (BITS 4 or 8, symmetric unless asym; keephead leaves the output
        projection (lm_head) in 32 bits)   |   dyn8: dynamic int8 per channel (activations too)   |   fp16: weights stored as half floats
Linear layers are turned from Gemm into MatMul first (the quantiser skips Gemm)."""
import os, sys, shutil, numpy as np, onnx
from onnx import helper, numpy_helper, TensorProto
part, scheme, name = sys.argv[1:4]
root = os.path.expanduser('~/hw-bench/glm_onnx'); dst = f'{root}/parts/{name}'
shutil.rmtree(dst, ignore_errors=True); os.makedirs(dst)
base = f'{root}/tmp/{part}_mm.onnx'
if not os.path.exists(base):   # the fp32 graph without the exporter's stale shape notes, linear layers as MatMul + Add
    os.makedirs(f'{root}/tmp', exist_ok=True)
    m = onnx.load(f'{root}/fp32/{part}.onnx'); del m.graph.value_info[:]
    inits = {i.name: i for i in m.graph.initializer}; new = []
    for n in m.graph.node:
        if n.op_type == 'Gemm' and n.input[1] in inits:
            a = {x.name: helper.get_attribute_value(x) for x in n.attribute}
            w = numpy_helper.to_array(inits[n.input[1]]); w = w.T if a.get('transB', 0) else w
            wn = n.input[1] + '_t'; m.graph.initializer.append(numpy_helper.from_array(np.ascontiguousarray(w), wn))
            mm = n.output[0] + '_mm' if len(n.input) > 2 else n.output[0]
            new.append(helper.make_node('MatMul', [n.input[0], wn], [mm], name=n.name + '_mm'))
            if len(n.input) > 2: new.append(helper.make_node('Add', [mm, n.input[2]], [n.output[0]], name=n.name + '_add'))
        else: new.append(n)
    del m.graph.node[:]; m.graph.node.extend(new)
    used = {i for n in m.graph.node for i in n.input}; keep = [i for i in m.graph.initializer if i.name in used]
    del m.graph.initializer[:]; m.graph.initializer.extend(keep)
    onnx.save_model(m, base, save_as_external_data=True, all_tensors_to_one_file=True, location=f'{part}_mm.onnx.data')
m = onnx.load(base)
kind = scheme.split(':')[0]
if kind == 'nb':
    from onnxruntime.quantization.matmul_nbits_quantizer import MatMulNBitsQuantizer
    f = scheme.split(':'); bits, block = int(f[1]), int(f[2]); asym = 'asym' in f
    exclude = []
    if 'keephead' in f:
        out = [n for n in m.graph.node if 'logits' in n.output[0] or n.output[0] == 'logits']
        exclude = [n.name for n in out if n.op_type == 'MatMul']
        print('keeping', exclude)
    attn = []
    if 'mixed' in f:   # attention weights (qkv, proj) in 8 bits, the rest in `bits`: first all the others, then the attention ones
        sizes = {i.name: tuple(i.dims) for i in m.graph.initializer}
        attn = [n.name for n in m.graph.node if n.op_type == 'MatMul' and sizes.get(n.input[1]) in ((1024, 3072), (1024, 1024))]
        print('attention matmuls:', len(attn)); exclude = attn
    q = MatMulNBitsQuantizer(m, block_size=block, is_symmetric=not asym, bits=bits, nodes_to_exclude=exclude)
    q.process(); m = q.model.model
    if attn:
        q8 = MatMulNBitsQuantizer(m, block_size=block, is_symmetric=not asym, bits=8); q8.process(); q = q8
    q.model.save_model_to_file(f'{dst}/{part}.onnx', use_external_data_format=True)
elif kind == 'dyn8':
    from onnxruntime.quantization import quantize_dynamic, QuantType
    quantize_dynamic(base, f'{dst}/{part}.onnx', weight_type=QuantType.QInt8, per_channel=True, op_types_to_quantize=['MatMul'], use_external_data_format=True)
elif kind == 'fp16':
    from onnxruntime.transformers.float16 import convert_float_to_float16
    m16 = convert_float_to_float16(m, keep_io_types=True, disable_shape_infer=True)
    onnx.save_model(m16, f'{dst}/{part}.onnx', save_as_external_data=True, all_tensors_to_one_file=True, location=f'{part}.onnx.data')
print(name, part, [(fn, os.path.getsize(f'{dst}/{fn}') // 2**20) for fn in sorted(os.listdir(dst))])