File size: 1,564 Bytes
0e322d7 49c3379 0e322d7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | """Where does a prompt block (largest S) spend its time: NPU calls vs expert rebinding vs the rest?
usage: python prof_block.py ENGINE_DIR"""
import sys, time
import numpy as np
import openvino as ov
from onw.runtime import SegmentedModel
def main():
core = ov.Core()
d = sys.argv[1]
m = SegmentedModel(core, d, "NPU", {"CACHE_DIR": d + "/npu_cache"})
S = m.S
acc = {"infer": 0.0, "set_tensor": 0.0, "n_set": 0}
wrapped = []
for req, meta, names in m.segs[S]:
class Pxy:
pass
p = Pxy()
p.model_inputs = req.model_inputs
def st(*a, _f=req.set_tensor):
t = time.perf_counter(); _f(*a); acc["set_tensor"] += time.perf_counter() - t; acc["n_set"] += 1
def inf(*a, _f=req.infer, **k):
t = time.perf_counter(); r = _f(*a, **k); acc["infer"] += time.perf_counter() - t; return r
p.set_tensor, p.infer = st, inf
wrapped.append((p, meta, names))
m.segs[S] = wrapped
rng = np.random.default_rng(0)
m.step(list(rng.integers(1000, 50000, S)))
for k in acc:
acc[k] = 0
n = 5
t0 = time.perf_counter()
for _ in range(n):
m.reset()
m.step(list(rng.integers(1000, 50000, S)))
tot = (time.perf_counter() - t0) / n * 1000
print(f"S={S} block: {tot:.0f} ms = NPU {acc['infer'] / n * 1000:.0f} ms + set_tensor {acc['set_tensor'] / n * 1000:.0f} ms "
f"({acc['n_set'] // n} calls) + other {tot - (acc['infer'] + acc['set_tensor']) / n * 1000:.0f} ms")
if __name__ == "__main__":
main()
|