bevdet-p150 / code /scripts /profile_ops.py

tt-model push bevdet-p150 (container)

d160c10 verified 11 days ago

10 kB

#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
"""Device profile of BEVDet's traced replays, one signposted range per trace segment (OPT_BASELINE.md).
Tracy + device profiler (absolute ``-o`` under ``$ROOT/generated/profiler``, never inside the bundle):
    bin/devrun -t 3600 -- python -m tracy -r -p -v --no-web-server --op-support-count 4000 \\
        -o $ROOT/generated/profiler/bevdet_baseline code/scripts/profile_ops.py
    tt-perf-report <reports/<ts>/ops_perf_results_<ts>.csv> --start-signpost encode --end-signpost encode_end
    python -m tt_bevdet.ttaw.profiling <reports dir> --start encode --end encode_end     # ops / kernel sum / gaps
ttnn-visualizer memory + graph report of one eager forward (slow; never used for timing):
    bin/devrun -t 3600 -- python code/scripts/profile_ops.py --graph-report $ROOT/generated/ttnn_reports/bevdet_baseline
    ttnn-visualizer --profiler-path $ROOT/generated/ttnn_reports/bevdet_baseline \\
        --performance-path $ROOT/generated/profiler/bevdet_baseline/reports/<ts>
The model is loaded like the API (every variant captured), three synthetic frames bring the stream (and its history
ring) to the steady state, the profiler buffer is flushed, then each variant is replayed once between its own
signposts (``encode`` .. ``encode_end``, ``decode``, ``push``, ``fill``; all inside ``trace`` .. ``trace_end``) with
``ttnn.ReadDeviceProfiler`` after each one (the device buffer holds ~1000 programs). A steady frame is
encode + decode + push; a reset or first frame runs fill instead of push. ``--rounds 2`` repeats the window
(``trace2`` / ``encode2`` ...) to check that the profile is stable; ``--eager`` adds one eager (untraced, program
cache hot) encode + decode between ``eager_encode`` / ``eager_decode`` signposts (op-to-op gaps of eager dispatch).
"""
from __future__ import annotations

import argparse
import json
import sys
import time
from pathlib import Path

HERE = Path(__file__).resolve()
sys.path.insert(0, str(HERE.parents[1]))

VARIANTS = ("encode", "decode", "push", "fill")


class _Marked:
    """Transparent wrapper of a callable module object: emits a Tracy signpost, then calls it (``--eager`` stage
    markers; host-side only, nothing changes on the device)."""

    def __init__(self, obj, name: str, signpost, after: str = ""):
        self._obj, self._name, self._signpost, self._after = obj, name, signpost, after

    def __call__(self, *a, **k):
        if self._name:
            self._signpost(self._name)
        out = self._obj(*a, **k)
        if self._after:
            self._signpost(self._after)
        return out

    def __getattr__(self, item):
        return getattr(self._obj, item)


def mark_stages(tt, signpost):
    """Signposts at the module boundaries of the encode / decode functions (one eager run). Returns an undo
    callable. Labels: enc.stem, enc.layer1..4, enc.neck, enc.depthnet.reduce, enc.depthnet.blocks,
    enc.depthnet.aspp, enc.depthnet.head, enc.out (the depth | feat concat); dec.gather, dec.layer1..3, dec.neck,
    dec.head."""
    undo = []

    def set_attr(obj, name, value):
        had = name in obj.__dict__
        old = obj.__dict__.get(name)
        setattr(obj, name, value)
        undo.append(lambda: setattr(obj, name, old) if had else obj.__dict__.pop(name, None))

    enc, dec = tt.encoder, tt.decoder
    set_attr(enc, "stem", _Marked(enc.stem, "enc.stem", signpost))
    for s, blocks in enumerate(enc.stages.stages):
        old = blocks[0]
        blocks[0] = _Marked(old, f"enc.layer{s + 1}", signpost)
        undo.append(lambda blocks=blocks, old=old: blocks.__setitem__(0, old))
    neck = enc.neck
    set_attr(enc, "neck", lambda *a, **k: (signpost("enc.neck"), neck(*a, **k))[1])
    set_attr(enc, "reduce", _Marked(enc.reduce, "enc.depthnet.reduce", signpost))
    c1, c2 = enc.blocks[0]
    enc.blocks[0] = (_Marked(c1, "enc.depthnet.blocks", signpost), c2)
    undo.append(lambda: enc.blocks.__setitem__(0, (c1, c2)))
    set_attr(enc, "aspp1", _Marked(enc.aspp1, "enc.depthnet.aspp", signpost))
    set_attr(enc, "logits", _Marked(enc.logits, "enc.depthnet.head", signpost))
    set_attr(tt, "encoder", _Marked(enc, "", signpost, after="enc.out"))
    gather = tt.fusion.gather
    set_attr(tt.fusion, "gather", lambda *a, **k: (signpost("dec.gather"), gather(*a, **k))[1])
    for s, blocks in enumerate(dec.backbone.stages):
        old = blocks[0]
        blocks[0] = _Marked(old, f"dec.layer{s + 1}", signpost)
        undo.append(lambda blocks=blocks, old=old: blocks.__setitem__(0, old))
    upsample = dec.upsample
    set_attr(dec, "upsample", lambda x, f: (signpost("dec.neck") if f == 4 else None, upsample(x, f))[1])
    set_attr(dec, "shared", _Marked(dec.shared, "dec.head", signpost))

    def restore():
        for fn in reversed(undo):
            fn()
    return restore


def steady_inputs(model):
    """Host values of one synthetic frame for ``run_eager``: the encode inputs / params and the decode inputs."""
    import numpy as np

    from tt_bevdet.tt.bev_fusion import history_grid
    from tt_bevdet.tt.warmup import synthetic_request

    req = synthetic_request(model.cfg)
    req["stream"] = {"id": "profile-eager"}
    prepared = model._prepare(**req)
    g_ctx, g_depth = prepared["gates"]
    enc_params = {"g_ctx": g_ctx.reshape(-1, 1, 1, g_ctx.shape[-1]),
                  "g_depth": g_depth.reshape(-1, 1, 1, g_depth.shape[-1])}
    plan = prepared["plan"]
    grid = history_grid(plan.transforms, plan.history, plan.flag, model.cfg.bev_hw)
    curr = np.zeros(model.tt.curr_shape, np.float32)
    prepared["state"].abort_frame()
    model._forget_stream("profile-eager")
    return dict(prepared["enc_in"]), enc_params, {"curr": curr}, {"grid": grid}


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
    ap.add_argument("--rounds", type=int, default=1)
    ap.add_argument("--eager", action="store_true")
    ap.add_argument("--graph-report", default=None, help="write a ttnn-visualizer memory/graph report of one eager "
                                                         "encode + decode to this directory instead of profiling")
    ap.add_argument("--dispatch", default=None, choices=[None, "eth", "worker"])
    ap.add_argument("--num-cqs", type=int, default=None)
    ap.add_argument("--weights-dir", default=None)
    ap.add_argument("--json", default=None, help="write the run's metadata (config, trace description)")
    a = ap.parse_args()
    import ttnn

    from tt_bevdet import BEVDet
    from tt_bevdet.tt.warmup import synthetic_images
    from tt_bevdet.ttaw.profiling import read_device_profiler, signpost

    t0 = time.perf_counter()
    model = BEVDet.from_pretrained(dispatch=a.dispatch, num_command_queues=a.num_cqs, weights_dir=a.weights_dir)
    meta = {"config": model.device_info, "knobs": model.knobs.as_dict(), "load_s": round(time.perf_counter() - t0, 1)}
    print("config:", json.dumps(meta["config"]), flush=True)
    dev, r = model.device, model.runner
    sync = lambda: ttnn.synchronize_device(dev)  # noqa: E731
    try:
        from tt_bevdet.host.inputs import CALIB_DIR
        from tt_bevdet.ttaw import io as tio

        calib = tio.resolve_calibration({"preset": "pandaset_019"}, CALIB_DIR)
        for s in range(3):                      # steady state: a fresh stream fills, then two pushes
            model(images=synthetic_images(model.cfg, seed=s + 1), calibration=calib, stream={"id": "profile"})
        sync()
        if a.graph_report:
            out_dir = Path(a.graph_report)
            out_dir.mkdir(parents=True, exist_ok=True)
            enc_in, enc_p, dec_in, dec_p = steady_inputs(model)
            r.run_eager("encode", enc_in, enc_p)        # program cache hot, nothing compiles inside the capture
            r.run_eager("decode", dec_in, dec_p)
            sync()
            t1 = time.perf_counter()
            with ttnn.graph.full_graph_capture(str(out_dir / "graph_capture.json")):
                r.run_eager("encode", enc_in, enc_p)
                r.run_eager("decode", dec_in, dec_p)
                sync()
            meta["graph_capture_s"] = round(time.perf_counter() - t1, 1)
            from ttnn import graph_report

            db = graph_report.import_report(out_dir, out_dir)
            ttnn.save_config_to_json_file(str(out_dir / "config.json"))
            meta["graph_report"] = {"dir": str(out_dir), "db": str(db)}
            print("graph report:", json.dumps(meta["graph_report"]), flush=True)
        else:
            read_device_profiler(dev)            # drop the load / warm-up data
            for k in range(1, a.rounds + 1):
                tag = "" if k == 1 else str(k)
                signpost(f"trace{tag}")
                for v in VARIANTS + (("push_x",) if getattr(model.tt, "push_x", False) else ()):
                    signpost(f"{v}{tag}")
                    r.replay(v)
                    sync()
                    signpost(f"{v}{tag}_end")
                    read_device_profiler(dev)
                signpost(f"trace{tag}_end")
            if a.eager:
                enc_in, enc_p, dec_in, dec_p = steady_inputs(model)
                restore = mark_stages(model.tt, signpost)
                try:
                    signpost("eager")
                    for v, (i, p) in (("encode", (enc_in, enc_p)), ("decode", (dec_in, dec_p))):
                        signpost(f"eager_{v}")
                        r.run_eager(v, i, p)
                        sync()
                        signpost(f"eager_{v}_end")
                        read_device_profiler(dev)
                    signpost("eager_end")
                finally:
                    restore()
        meta["trace"] = r.describe()
    finally:
        model.close()
    if a.json:
        Path(a.json).parent.mkdir(parents=True, exist_ok=True)
        Path(a.json).write_text(json.dumps(meta, indent=1, default=str) + "\n")
    return 0


if __name__ == "__main__":
    sys.exit(main())

	#!/usr/bin/env python3
	# SPDX-License-Identifier: Apache-2.0
	"""Device profile of BEVDet's traced replays, one signposted range per trace segment (OPT_BASELINE.md).

	Tracy + device profiler (absolute ``-o`` under ``$ROOT/generated/profiler``, never inside the bundle):

	bin/devrun -t 3600 -- python -m tracy -r -p -v --no-web-server --op-support-count 4000 \\
	-o $ROOT/generated/profiler/bevdet_baseline code/scripts/profile_ops.py
	tt-perf-report <reports/<ts>/ops_perf_results_<ts>.csv> --start-signpost encode --end-signpost encode_end
	python -m tt_bevdet.ttaw.profiling <reports dir> --start encode --end encode_end # ops / kernel sum / gaps

	ttnn-visualizer memory + graph report of one eager forward (slow; never used for timing):

	bin/devrun -t 3600 -- python code/scripts/profile_ops.py --graph-report $ROOT/generated/ttnn_reports/bevdet_baseline
	ttnn-visualizer --profiler-path $ROOT/generated/ttnn_reports/bevdet_baseline \\
	--performance-path $ROOT/generated/profiler/bevdet_baseline/reports/<ts>

	The model is loaded like the API (every variant captured), three synthetic frames bring the stream (and its history
	ring) to the steady state, the profiler buffer is flushed, then each variant is replayed once between its own
	signposts (``encode`` .. ``encode_end``, ``decode``, ``push``, ``fill``; all inside ``trace`` .. ``trace_end``) with
	``ttnn.ReadDeviceProfiler`` after each one (the device buffer holds ~1000 programs). A steady frame is
	encode + decode + push; a reset or first frame runs fill instead of push. ``--rounds 2`` repeats the window
	(``trace2`` / ``encode2`` ...) to check that the profile is stable; ``--eager`` adds one eager (untraced, program
	cache hot) encode + decode between ``eager_encode`` / ``eager_decode`` signposts (op-to-op gaps of eager dispatch).
	"""
	from __future__ import annotations

	import argparse
	import json
	import sys
	import time
	from pathlib import Path

	HERE = Path(__file__).resolve()
	sys.path.insert(0, str(HERE.parents[1]))

	VARIANTS = ("encode", "decode", "push", "fill")


	class _Marked:
	"""Transparent wrapper of a callable module object: emits a Tracy signpost, then calls it (``--eager`` stage
	markers; host-side only, nothing changes on the device)."""

	def __init__(self, obj, name: str, signpost, after: str = ""):
	self._obj, self._name, self._signpost, self._after = obj, name, signpost, after

	def __call__(self, a, *k):
	if self._name:
	self._signpost(self._name)
	out = self._obj(a, *k)
	if self._after:
	self._signpost(self._after)
	return out

	def __getattr__(self, item):
	return getattr(self._obj, item)


	def mark_stages(tt, signpost):
	"""Signposts at the module boundaries of the encode / decode functions (one eager run). Returns an undo
	callable. Labels: enc.stem, enc.layer1..4, enc.neck, enc.depthnet.reduce, enc.depthnet.blocks,
	enc.depthnet.aspp, enc.depthnet.head, enc.out (the depth \| feat concat); dec.gather, dec.layer1..3, dec.neck,
	dec.head."""
	undo = []

	def set_attr(obj, name, value):
	had = name in obj.__dict__
	old = obj.__dict__.get(name)
	setattr(obj, name, value)
	undo.append(lambda: setattr(obj, name, old) if had else obj.__dict__.pop(name, None))

	enc, dec = tt.encoder, tt.decoder
	set_attr(enc, "stem", _Marked(enc.stem, "enc.stem", signpost))
	for s, blocks in enumerate(enc.stages.stages):
	old = blocks[0]
	blocks[0] = _Marked(old, f"enc.layer{s + 1}", signpost)
	undo.append(lambda blocks=blocks, old=old: blocks.__setitem__(0, old))
	neck = enc.neck
	set_attr(enc, "neck", lambda a, k: (signpost("enc.neck"), neck(a, **k))[1])
	set_attr(enc, "reduce", _Marked(enc.reduce, "enc.depthnet.reduce", signpost))
	c1, c2 = enc.blocks[0]
	enc.blocks[0] = (_Marked(c1, "enc.depthnet.blocks", signpost), c2)
	undo.append(lambda: enc.blocks.__setitem__(0, (c1, c2)))
	set_attr(enc, "aspp1", _Marked(enc.aspp1, "enc.depthnet.aspp", signpost))
	set_attr(enc, "logits", _Marked(enc.logits, "enc.depthnet.head", signpost))
	set_attr(tt, "encoder", _Marked(enc, "", signpost, after="enc.out"))
	gather = tt.fusion.gather
	set_attr(tt.fusion, "gather", lambda a, k: (signpost("dec.gather"), gather(a, **k))[1])
	for s, blocks in enumerate(dec.backbone.stages):
	old = blocks[0]
	blocks[0] = _Marked(old, f"dec.layer{s + 1}", signpost)
	undo.append(lambda blocks=blocks, old=old: blocks.__setitem__(0, old))
	upsample = dec.upsample
	set_attr(dec, "upsample", lambda x, f: (signpost("dec.neck") if f == 4 else None, upsample(x, f))[1])
	set_attr(dec, "shared", _Marked(dec.shared, "dec.head", signpost))

	def restore():
	for fn in reversed(undo):
	fn()
	return restore


	def steady_inputs(model):
	"""Host values of one synthetic frame for ``run_eager``: the encode inputs / params and the decode inputs."""
	import numpy as np

	from tt_bevdet.tt.bev_fusion import history_grid
	from tt_bevdet.tt.warmup import synthetic_request

	req = synthetic_request(model.cfg)
	req["stream"] = {"id": "profile-eager"}
	prepared = model._prepare(**req)
	g_ctx, g_depth = prepared["gates"]
	enc_params = {"g_ctx": g_ctx.reshape(-1, 1, 1, g_ctx.shape[-1]),
	"g_depth": g_depth.reshape(-1, 1, 1, g_depth.shape[-1])}
	plan = prepared["plan"]
	grid = history_grid(plan.transforms, plan.history, plan.flag, model.cfg.bev_hw)
	curr = np.zeros(model.tt.curr_shape, np.float32)
	prepared["state"].abort_frame()
	model._forget_stream("profile-eager")
	return dict(prepared["enc_in"]), enc_params, {"curr": curr}, {"grid": grid}


	def main() -> int:
	ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
	ap.add_argument("--rounds", type=int, default=1)
	ap.add_argument("--eager", action="store_true")
	ap.add_argument("--graph-report", default=None, help="write a ttnn-visualizer memory/graph report of one eager "
	"encode + decode to this directory instead of profiling")
	ap.add_argument("--dispatch", default=None, choices=[None, "eth", "worker"])
	ap.add_argument("--num-cqs", type=int, default=None)
	ap.add_argument("--weights-dir", default=None)
	ap.add_argument("--json", default=None, help="write the run's metadata (config, trace description)")
	a = ap.parse_args()
	import ttnn

	from tt_bevdet import BEVDet
	from tt_bevdet.tt.warmup import synthetic_images
	from tt_bevdet.ttaw.profiling import read_device_profiler, signpost

	t0 = time.perf_counter()
	model = BEVDet.from_pretrained(dispatch=a.dispatch, num_command_queues=a.num_cqs, weights_dir=a.weights_dir)
	meta = {"config": model.device_info, "knobs": model.knobs.as_dict(), "load_s": round(time.perf_counter() - t0, 1)}
	print("config:", json.dumps(meta["config"]), flush=True)
	dev, r = model.device, model.runner
	sync = lambda: ttnn.synchronize_device(dev) # noqa: E731
	try:
	from tt_bevdet.host.inputs import CALIB_DIR
	from tt_bevdet.ttaw import io as tio

	calib = tio.resolve_calibration({"preset": "pandaset_019"}, CALIB_DIR)
	for s in range(3): # steady state: a fresh stream fills, then two pushes
	model(images=synthetic_images(model.cfg, seed=s + 1), calibration=calib, stream={"id": "profile"})
	sync()
	if a.graph_report:
	out_dir = Path(a.graph_report)
	out_dir.mkdir(parents=True, exist_ok=True)
	enc_in, enc_p, dec_in, dec_p = steady_inputs(model)
	r.run_eager("encode", enc_in, enc_p) # program cache hot, nothing compiles inside the capture
	r.run_eager("decode", dec_in, dec_p)
	sync()
	t1 = time.perf_counter()
	with ttnn.graph.full_graph_capture(str(out_dir / "graph_capture.json")):
	r.run_eager("encode", enc_in, enc_p)
	r.run_eager("decode", dec_in, dec_p)
	sync()
	meta["graph_capture_s"] = round(time.perf_counter() - t1, 1)
	from ttnn import graph_report

	db = graph_report.import_report(out_dir, out_dir)
	ttnn.save_config_to_json_file(str(out_dir / "config.json"))
	meta["graph_report"] = {"dir": str(out_dir), "db": str(db)}
	print("graph report:", json.dumps(meta["graph_report"]), flush=True)
	else:
	read_device_profiler(dev) # drop the load / warm-up data
	for k in range(1, a.rounds + 1):
	tag = "" if k == 1 else str(k)
	signpost(f"trace{tag}")
	for v in VARIANTS + (("push_x",) if getattr(model.tt, "push_x", False) else ()):
	signpost(f"{v}{tag}")
	r.replay(v)
	sync()
	signpost(f"{v}{tag}_end")
	read_device_profiler(dev)
	signpost(f"trace{tag}_end")
	if a.eager:
	enc_in, enc_p, dec_in, dec_p = steady_inputs(model)
	restore = mark_stages(model.tt, signpost)
	try:
	signpost("eager")
	for v, (i, p) in (("encode", (enc_in, enc_p)), ("decode", (dec_in, dec_p))):
	signpost(f"eager_{v}")
	r.run_eager(v, i, p)
	sync()
	signpost(f"eager_{v}_end")
	read_device_profiler(dev)
	signpost("eager_end")
	finally:
	restore()
	meta["trace"] = r.describe()
	finally:
	model.close()
	if a.json:
	Path(a.json).parent.mkdir(parents=True, exist_ok=True)
	Path(a.json).write_text(json.dumps(meta, indent=1, default=str) + "\n")
	return 0


	if __name__ == "__main__":
	sys.exit(main())