File size: 10,035 Bytes
77e3c67 d160c10 77e3c67 d160c10 77e3c67 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 | #!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
"""Device profile of BEVDet's traced replays, one signposted range per trace segment (OPT_BASELINE.md).
Tracy + device profiler (absolute ``-o`` under ``$ROOT/generated/profiler``, never inside the bundle):
bin/devrun -t 3600 -- python -m tracy -r -p -v --no-web-server --op-support-count 4000 \\
-o $ROOT/generated/profiler/bevdet_baseline code/scripts/profile_ops.py
tt-perf-report <reports/<ts>/ops_perf_results_<ts>.csv> --start-signpost encode --end-signpost encode_end
python -m tt_bevdet.ttaw.profiling <reports dir> --start encode --end encode_end # ops / kernel sum / gaps
ttnn-visualizer memory + graph report of one eager forward (slow; never used for timing):
bin/devrun -t 3600 -- python code/scripts/profile_ops.py --graph-report $ROOT/generated/ttnn_reports/bevdet_baseline
ttnn-visualizer --profiler-path $ROOT/generated/ttnn_reports/bevdet_baseline \\
--performance-path $ROOT/generated/profiler/bevdet_baseline/reports/<ts>
The model is loaded like the API (every variant captured), three synthetic frames bring the stream (and its history
ring) to the steady state, the profiler buffer is flushed, then each variant is replayed once between its own
signposts (``encode`` .. ``encode_end``, ``decode``, ``push``, ``fill``; all inside ``trace`` .. ``trace_end``) with
``ttnn.ReadDeviceProfiler`` after each one (the device buffer holds ~1000 programs). A steady frame is
encode + decode + push; a reset or first frame runs fill instead of push. ``--rounds 2`` repeats the window
(``trace2`` / ``encode2`` ...) to check that the profile is stable; ``--eager`` adds one eager (untraced, program
cache hot) encode + decode between ``eager_encode`` / ``eager_decode`` signposts (op-to-op gaps of eager dispatch).
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
HERE = Path(__file__).resolve()
sys.path.insert(0, str(HERE.parents[1]))
VARIANTS = ("encode", "decode", "push", "fill")
class _Marked:
"""Transparent wrapper of a callable module object: emits a Tracy signpost, then calls it (``--eager`` stage
markers; host-side only, nothing changes on the device)."""
def __init__(self, obj, name: str, signpost, after: str = ""):
self._obj, self._name, self._signpost, self._after = obj, name, signpost, after
def __call__(self, *a, **k):
if self._name:
self._signpost(self._name)
out = self._obj(*a, **k)
if self._after:
self._signpost(self._after)
return out
def __getattr__(self, item):
return getattr(self._obj, item)
def mark_stages(tt, signpost):
"""Signposts at the module boundaries of the encode / decode functions (one eager run). Returns an undo
callable. Labels: enc.stem, enc.layer1..4, enc.neck, enc.depthnet.reduce, enc.depthnet.blocks,
enc.depthnet.aspp, enc.depthnet.head, enc.out (the depth | feat concat); dec.gather, dec.layer1..3, dec.neck,
dec.head."""
undo = []
def set_attr(obj, name, value):
had = name in obj.__dict__
old = obj.__dict__.get(name)
setattr(obj, name, value)
undo.append(lambda: setattr(obj, name, old) if had else obj.__dict__.pop(name, None))
enc, dec = tt.encoder, tt.decoder
set_attr(enc, "stem", _Marked(enc.stem, "enc.stem", signpost))
for s, blocks in enumerate(enc.stages.stages):
old = blocks[0]
blocks[0] = _Marked(old, f"enc.layer{s + 1}", signpost)
undo.append(lambda blocks=blocks, old=old: blocks.__setitem__(0, old))
neck = enc.neck
set_attr(enc, "neck", lambda *a, **k: (signpost("enc.neck"), neck(*a, **k))[1])
set_attr(enc, "reduce", _Marked(enc.reduce, "enc.depthnet.reduce", signpost))
c1, c2 = enc.blocks[0]
enc.blocks[0] = (_Marked(c1, "enc.depthnet.blocks", signpost), c2)
undo.append(lambda: enc.blocks.__setitem__(0, (c1, c2)))
set_attr(enc, "aspp1", _Marked(enc.aspp1, "enc.depthnet.aspp", signpost))
set_attr(enc, "logits", _Marked(enc.logits, "enc.depthnet.head", signpost))
set_attr(tt, "encoder", _Marked(enc, "", signpost, after="enc.out"))
gather = tt.fusion.gather
set_attr(tt.fusion, "gather", lambda *a, **k: (signpost("dec.gather"), gather(*a, **k))[1])
for s, blocks in enumerate(dec.backbone.stages):
old = blocks[0]
blocks[0] = _Marked(old, f"dec.layer{s + 1}", signpost)
undo.append(lambda blocks=blocks, old=old: blocks.__setitem__(0, old))
upsample = dec.upsample
set_attr(dec, "upsample", lambda x, f: (signpost("dec.neck") if f == 4 else None, upsample(x, f))[1])
set_attr(dec, "shared", _Marked(dec.shared, "dec.head", signpost))
def restore():
for fn in reversed(undo):
fn()
return restore
def steady_inputs(model):
"""Host values of one synthetic frame for ``run_eager``: the encode inputs / params and the decode inputs."""
import numpy as np
from tt_bevdet.tt.bev_fusion import history_grid
from tt_bevdet.tt.warmup import synthetic_request
req = synthetic_request(model.cfg)
req["stream"] = {"id": "profile-eager"}
prepared = model._prepare(**req)
g_ctx, g_depth = prepared["gates"]
enc_params = {"g_ctx": g_ctx.reshape(-1, 1, 1, g_ctx.shape[-1]),
"g_depth": g_depth.reshape(-1, 1, 1, g_depth.shape[-1])}
plan = prepared["plan"]
grid = history_grid(plan.transforms, plan.history, plan.flag, model.cfg.bev_hw)
curr = np.zeros(model.tt.curr_shape, np.float32)
prepared["state"].abort_frame()
model._forget_stream("profile-eager")
return dict(prepared["enc_in"]), enc_params, {"curr": curr}, {"grid": grid}
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
ap.add_argument("--rounds", type=int, default=1)
ap.add_argument("--eager", action="store_true")
ap.add_argument("--graph-report", default=None, help="write a ttnn-visualizer memory/graph report of one eager "
"encode + decode to this directory instead of profiling")
ap.add_argument("--dispatch", default=None, choices=[None, "eth", "worker"])
ap.add_argument("--num-cqs", type=int, default=None)
ap.add_argument("--weights-dir", default=None)
ap.add_argument("--json", default=None, help="write the run's metadata (config, trace description)")
a = ap.parse_args()
import ttnn
from tt_bevdet import BEVDet
from tt_bevdet.tt.warmup import synthetic_images
from tt_bevdet.ttaw.profiling import read_device_profiler, signpost
t0 = time.perf_counter()
model = BEVDet.from_pretrained(dispatch=a.dispatch, num_command_queues=a.num_cqs, weights_dir=a.weights_dir)
meta = {"config": model.device_info, "knobs": model.knobs.as_dict(), "load_s": round(time.perf_counter() - t0, 1)}
print("config:", json.dumps(meta["config"]), flush=True)
dev, r = model.device, model.runner
sync = lambda: ttnn.synchronize_device(dev) # noqa: E731
try:
from tt_bevdet.host.inputs import CALIB_DIR
from tt_bevdet.ttaw import io as tio
calib = tio.resolve_calibration({"preset": "pandaset_019"}, CALIB_DIR)
for s in range(3): # steady state: a fresh stream fills, then two pushes
model(images=synthetic_images(model.cfg, seed=s + 1), calibration=calib, stream={"id": "profile"})
sync()
if a.graph_report:
out_dir = Path(a.graph_report)
out_dir.mkdir(parents=True, exist_ok=True)
enc_in, enc_p, dec_in, dec_p = steady_inputs(model)
r.run_eager("encode", enc_in, enc_p) # program cache hot, nothing compiles inside the capture
r.run_eager("decode", dec_in, dec_p)
sync()
t1 = time.perf_counter()
with ttnn.graph.full_graph_capture(str(out_dir / "graph_capture.json")):
r.run_eager("encode", enc_in, enc_p)
r.run_eager("decode", dec_in, dec_p)
sync()
meta["graph_capture_s"] = round(time.perf_counter() - t1, 1)
from ttnn import graph_report
db = graph_report.import_report(out_dir, out_dir)
ttnn.save_config_to_json_file(str(out_dir / "config.json"))
meta["graph_report"] = {"dir": str(out_dir), "db": str(db)}
print("graph report:", json.dumps(meta["graph_report"]), flush=True)
else:
read_device_profiler(dev) # drop the load / warm-up data
for k in range(1, a.rounds + 1):
tag = "" if k == 1 else str(k)
signpost(f"trace{tag}")
for v in VARIANTS + (("push_x",) if getattr(model.tt, "push_x", False) else ()):
signpost(f"{v}{tag}")
r.replay(v)
sync()
signpost(f"{v}{tag}_end")
read_device_profiler(dev)
signpost(f"trace{tag}_end")
if a.eager:
enc_in, enc_p, dec_in, dec_p = steady_inputs(model)
restore = mark_stages(model.tt, signpost)
try:
signpost("eager")
for v, (i, p) in (("encode", (enc_in, enc_p)), ("decode", (dec_in, dec_p))):
signpost(f"eager_{v}")
r.run_eager(v, i, p)
sync()
signpost(f"eager_{v}_end")
read_device_profiler(dev)
signpost("eager_end")
finally:
restore()
meta["trace"] = r.describe()
finally:
model.close()
if a.json:
Path(a.json).parent.mkdir(parents=True, exist_ok=True)
Path(a.json).write_text(json.dumps(meta, indent=1, default=str) + "\n")
return 0
if __name__ == "__main__":
sys.exit(main())
|