File size: 9,879 Bytes
3699e02
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
"""Cuts the handwriting reader's 16-bit vision network (vision-gpu.onnx, 868 MB) into parts that run one after the other.

Why: QNN on a phone compiles a session's whole graph at once, and for the full network that took the app past 3.3 GB on a 6 GB phone
(Android killed it). Each part is compiled on its own, one at a time, and kept as its own compiled file.

Every part takes the picture size the same way: the patch dimension of each input is named '4*m', as in the whole network, so fixing
that one name fixes the sizes of all parts. Between parts the values are float32 (the parts cast to and from 16 bits inside).

    python3 tool/split_glm_vision.py vision-gpu.onnx out_dir [parts: 1, 4 or 5 (the default)]

writes vision-gpu.part1.onnx (+ .data) ... and checks on the processor that the parts in a row give the whole network's answer.
"""

import heapq
import os
import sys

import numpy as np
import onnx
from onnx import TensorProto, helper

# the blocks each part ends after (24 blocks; the last part has the merger too, 118 MB). 'front': a part of the rotary tables alone (from
# pos_ids: GatherND, Cos, Sin; no weights), which QNN's GPU backend on a Snapdragon 865 compiled but could not run (error 6999): with it
# on its own, the app runs it on the processor and the four parts of weights on QNN
ENDS = {1: [23], 4: [6, 12, 18, 23], 5: ['front', 6, 12, 18, 23]}


def topo(g):
    prod = {o: i for i, n in enumerate(g.node) for o in n.output}
    deps = [set(prod[x] for x in n.input if x in prod) for n in g.node]
    users = [[] for _ in g.node]
    for i, d in enumerate(deps):
        for j in d:
            users[j].append(i)
    indeg = [len(d) for d in deps]
    h = [i for i in range(len(g.node)) if indeg[i] == 0]
    heapq.heapify(h)
    order = []
    while h:
        i = heapq.heappop(h)
        order.append(i)
        for u in users[i]:
            indeg[u] -= 1
            if indeg[u] == 0:
                heapq.heappush(h, u)
    assert len(order) == len(g.node)
    return [g.node[i] for i in order]


def main():
    src, out = sys.argv[1], sys.argv[2]
    count = int(sys.argv[3]) if len(sys.argv) > 3 else 5
    os.makedirs(out, exist_ok=True)
    stem = os.path.basename(src)[: -len('.onnx')]
    m = onnx.load(src)  # with the weights
    g = m.graph
    nodes = topo(g)
    inits = {t.name: t for t in g.initializer}
    ginputs = {i.name: i for i in g.input}
    # types of the values between parts
    bare = onnx.load(src, load_external_data=False)
    sorted_nodes = topo(bare.graph)
    bare.graph.ClearField('node')
    bare.graph.node.extend(sorted_nodes)  # (inference goes through the nodes in their order)
    typed = onnx.shape_inference.infer_shapes(bare)
    vtype = {v.name: v.type.tensor_type.elem_type for v in list(typed.graph.value_info) + list(typed.graph.input) + list(typed.graph.output)}

    pos = {o: k for k, n in enumerate(nodes) for o in n.output}
    for t in ginputs:
        pos[t] = -1
    last = {}
    for k, n in enumerate(nodes):
        for x in n.input:
            if x and x not in inits:
                last[x] = max(last.get(x, -1), k)
    for o in g.output:
        last[o.name] = len(nodes)

    def crossing(k):
        return [t for t, p in pos.items() if p <= k and last.get(t, -1) > k]

    # a cut a few nodes after each block's Softmax, where only the hidden state and the two rotary tables cross
    softmax = [k for k, n in enumerate(nodes) if n.op_type == 'Softmax']
    cuts = []
    for a, b in zip(softmax, softmax[1:] + [len(nodes) - 1]):
        cuts.append(min(range(a, b), key=lambda k: len(crossing(k))))
    # (the front: the nodes before the first that reads the picture; only the rotary tables cross there, besides the inputs)
    front = min(k for k, n in enumerate(nodes) if 'pixel_values' in n.input) - 1
    assert all(t in ginputs or t.startswith('unsqueeze') for t in crossing(front)), crossing(front)
    ends = [front if b == 'front' else cuts[b] for b in ENDS[count][:-1]] + [len(nodes) - 1]
    for k in ends[:-1]:
        missing = [t for t in crossing(k) if t not in vtype]
        assert not missing, f'no type known for {missing}'

    start = 0
    names = []
    for p, end in enumerate(ends):
        part = nodes[start:end + 1]
        ins = [t for t in crossing(start - 1)] if start > 0 else [i.name for i in g.input]
        made = {o for n in part for o in n.output}
        # (the rotary tables are made by the first part only: the later ones get them from it, as the caller keeps every output)
        outs = [t for t in crossing(end) if t in made] if end < len(nodes) - 1 else [o.name for o in g.output]
        used = {x for n in part for x in n.input}
        new_nodes, inputs, outputs = [], [], []
        for t in ins:
            if t in ginputs:
                inputs.append(ginputs[t])
                continue
            # float32 in, cast to the type inside
            inner = vtype.get(t, TensorProto.FLOAT)
            inputs.append(helper.make_tensor_value_info(t + '_f32' if inner != TensorProto.FLOAT else t, TensorProto.FLOAT, None))
            if inner != TensorProto.FLOAT:
                new_nodes.append(helper.make_node('Cast', [t + '_f32'], [t], to=inner, name='part_in_' + t))
        new_nodes += part
        for t in outs:
            if t in {o.name for o in g.output}:
                outputs.append([o for o in g.output if o.name == t][0])
                continue
            inner = vtype.get(t, TensorProto.FLOAT)
            if inner != TensorProto.FLOAT:
                new_nodes.append(helper.make_node('Cast', [t], [t + '_f32'], to=TensorProto.FLOAT, name='part_out_' + t))
            outputs.append(helper.make_tensor_value_info(t + '_f32' if inner != TensorProto.FLOAT else t, TensorProto.FLOAT, None))
        graph = helper.make_graph(new_nodes, f'{g.name}_part{p + 1}', inputs, outputs, [inits[x] for x in sorted(used) if x in inits])
        model = helper.make_model(graph, opset_imports=m.opset_import, ir_version=m.ir_version)
        name = f'{stem}.part{p + 1}.onnx'
        path = os.path.join(out, name)
        for f in (path, path + '.data'):
            if os.path.exists(f):
                os.remove(f)
        onnx.save(model, path, save_as_external_data=True, all_tensors_to_one_file=True, location=name + '.data', size_threshold=1024)
        names.append(name)
        print(name, 'nodes', len(part), 'in', [i.name for i in inputs], 'out', [o.name for o in outputs],
              f'{os.path.getsize(path + ".data") / 1e6:.0f} MB')
        start = end + 1
    name_dims(out, names)
    check(src, [os.path.join(out, n) for n in names])


def name_dims(out, names):
    """Shapes of the inputs and outputs between parts, from a run, with the patch dimension named '4*m' (fixed by the app per size)."""
    import onnxruntime as ort
    patches = 52  # (not a size the model has elsewhere: the rotary tables are 64 wide)
    feeds = {'pixel_values': np.random.rand(patches, 1176).astype(np.float32), 'pos_ids': np.stack([np.arange(patches) // 8, np.arange(patches) % 8], 1).astype(np.int64)}
    values = dict(feeds)
    shapes = {}
    for n in names:
        s = ort.InferenceSession(os.path.join(out, n), providers=['CPUExecutionProvider'])
        res = s.run(None, {i.name: values[i.name] for i in s.get_inputs()})
        for o, v in zip(s.get_outputs(), res):
            values[o.name] = v
            shapes[o.name] = v.shape
    whole = {'pixel_values', 'pos_ids', 'features'}  # (the network's own: their shapes are kept)
    for n in names:
        path = os.path.join(out, n)
        m = onnx.load(path, load_external_data=False)
        for v in list(m.graph.input) + list(m.graph.output):
            if v.name not in shapes or v.name in whole:
                continue
            dims = v.type.tensor_type.shape
            dims.ClearField('dim')
            for d in shapes[v.name]:
                dim = dims.dim.add()
                if d == patches:
                    dim.dim_param = '4*m'
                else:
                    dim.dim_value = d
        onnx.save(m, path)  # (the weights stay in the .data file, which is not rewritten)
        print(n, {v.name: [d.dim_param or d.dim_value for d in v.type.tensor_type.shape.dim] for v in list(m.graph.input) + list(m.graph.output)})


def check(src, paths):
    import onnxruntime as ort
    # (graph optimisations off. Not exactly the whole network's numbers: the processor works 16-bit operators out in 32 bits and keeps a
    # value in 32 bits from one operator to the next where it can, which a cut stops (a value between parts is rounded to 16 bits).
    # With 1 part (the sorted whole) the difference is 0; with 4 it was 1.4e-3 on values up to 0.4, the size of what turning ONNX
    # Runtime's own graph optimisations on or off changes in the whole network: 1.2e-3)
    o = ort.SessionOptions()
    o.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL
    whole = ort.InferenceSession(src, o, providers=['CPUExecutionProvider'])
    parts = [ort.InferenceSession(p, o, providers=['CPUExecutionProvider']) for p in paths]
    for patches in (16, 64, 200):
        rng = np.random.default_rng(patches)
        side = 8
        feeds = {'pixel_values': rng.random((patches, 1176), dtype=np.float32), 'pos_ids': np.stack([np.arange(patches) // side, np.arange(patches) % side], 1).astype(np.int64)}
        want = whole.run(None, feeds)[0]
        values = dict(feeds)
        for s in parts:
            for o, v in zip(s.get_outputs(), s.run(None, {i.name: values[i.name] for i in s.get_inputs()})):
                values[o.name] = v
        got = values['features']
        print(f'{patches} patches: features {got.shape}, largest difference from the whole network {np.abs(got - want).max():.3g} (values up to {np.abs(want).max():.3g})')


if __name__ == '__main__':
    main()