File size: 37,122 Bytes
51defdc e91e989 51defdc e91e989 8eeb550 e91e989 8eeb550 e91e989 8eeb550 e91e989 8eeb550 e91e989 8eeb550 e91e989 8eeb550 e91e989 51defdc e91e989 51defdc e91e989 51defdc e91e989 51defdc 8eeb550 e91e989 51defdc 8eeb550 51defdc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 | # SPDX-License-Identifier: Apache-2.0
"""Python API: METEOR (TIER IV, AutowareFoundation/meteor) on one Tenstorrent Blackhole p150.
from tt_meteor import METEOR, load_sample
with METEOR.from_pretrained(device_id=0) as model: # weights -> HF cache, device open, traces captured
out = model(**load_sample("code/tt_meteor/samples/synthetic_8cam.json")) # 8 cameras + calibration + speed
print(out.to_dict()) # the same JSON as POST /predict
The contract shared by every bundle of the Autoware collection is the vendored ``ttaw.api_base.ModelBase``
(BUNDLE_CONVENTIONS.md section 8): ``from_pretrained`` resolves the pinned weights before it claims the chip, opens it
(ETH dispatch, 12x10), builds the graph and captures every trace variant in ``warmup_variants``, so the first call is
as fast as the later ones; calls are serialised by a lock (one chip, batch 1) and fill ``timing_ms``; ``close()`` is
idempotent, also runs at interpreter exit, and closes the chip only if the model opened it. The HTTP server
(``tt_meteor.server.app``) calls this class, so ``/predict`` and ``model(...)`` agree bit for bit.
Inputs per call (``_prepare``; host code in ``tt_meteor.host``, METEOR's own runtime semantics: there is no Autoware
package):
- ``images``: the cameras ``CAM_FRONT_WIDE``, ``CAM_FRONT_LEFT``, ``CAM_FRONT_RIGHT``, ``CAM_BACK_WIDE``,
``CAM_BACK_LEFT``, ``CAM_BACK_RIGHT``, ``CAM_FRONT_NARROW``, ``CAM_BACK_NARROW`` (any order; raw, unrectified frames
of at least 768x432, resized with OpenCV INTER_AREA semantics); the two narrow cameras may be absent (zero image +
donor pose, as trained);
- ``calibration``: per camera ``intrinsics`` (of the image as sent) and ``T_ref_from_camera`` (camera optical frame ->
ego / base_link), or ``{"preset": name}``;
- ``ego_speed``: m/s (the graph's ``v0``);
- ``stream``: ``{"id", "reset", "timestamp_s", "T_world_from_ego"}`` (or ``"pose": [x, y, yaw]``): the host temporal
post-processing per stream (BEV seg fusion needs the pose; yaw smoothing; mode hysteresis).
Runtime params (``RUNTIME_PARAMS``): every knob of METEOR's C++ renderer (``host.postprocess.PostConfig``). Load-time
knobs: :data:`KNOBS` (``METEOR_*``): ``INPUT_NORM`` (``onnx`` = the released /255, D12; ``imagenet`` = the trained
normalisation), ``DEPTH_MEAN_BINS`` (``log`` as exported; ``linear``), ``MAX_STREAMS``.
The device graph is ``tt_meteor.tt.model.TtMETEOR``: the whole network in ONE trace (``frame``), replayed per call;
the host half (``_build_host``, ``_prepare``, ``_postprocess``) is what the host tests exercise. Importing this module
has no side effects (device code is imported in ``_build``).
"""
from __future__ import annotations
from collections import OrderedDict
from typing import Any, Dict, Mapping, Optional
from . import io as tio
from .device import DEVICE_DEFAULTS
from .host.outputs import LABELS as _LABELS
from .host.outputs import MeteorOutput
from .reference.config import CAMERAS
from .ttaw.api_base import ModelBase
from .ttaw.knobs import Knob, Knobs
__all__ = ["METEOR", "Output", "KNOBS", "load_sample"]
# The result class of this model (the multi-task MeteorOutput of host.outputs; BUNDLE_CONVENTIONS 7.4 "per head").
Output = MeteorOutput
# Load-time switches (read once at build; env METEOR_<NAME>). Each optimization adds its knob here (A/B switch).
KNOBS = Knobs("METEOR", [
Knob("INPUT_NORM", "onnx", "input normalisation: the released graph's /255 (D12 default, ORT parity) or the "
"trained ImageNet mean/std (SPEC section 10 risk 1)", choices=("onnx", "imagenet")),
Knob("DEPTH_MEAN_BINS", "log", "depth_mean bin centres: log-spaced as exported (parity) or linear 1 + 1.25 b "
"(the trained bins, SPEC section 10 risk 2)", choices=("log", "linear")),
Knob("MAX_STREAMS", 16, "host temporal states kept (seg fusion, yaw tracks, mode); the least recent is dropped"),
Knob("PROFILE", "accurate", "serve profile (D21): accurate = the release's precision (three-term image branch and "
"box refiner; bit-identical to the release, every frozen gate); fast = the reduced-"
"precision defaults of PROFILE_DEFAULTS (single-term bf16 image branch, bf16 weights "
"at HiFi2 in the image heads, single-term box refiner with fp32 weights at HiFi4), "
"judged by the D21 task-level gates (scripts/task_gates.py over "
"scripts/run_public_frames.py). A knob set explicitly overrides its profile default",
choices=("accurate", "fast")),
Knob("IMAGE_CONV", "HiFi4+fp32+l1acc:w=fp32", "precision of the image branch's single-term convs "
"(IMAGE_PRECISION=bf16; ttaw Precision spec)"),
Knob("IMAGE_STEM", "bf16", "IMAGE_PRECISION=bf16 only: fp32 = the stem conv writes fp32, the exact fp32 pool, and "
"the first ResNet block's conv1 (three-term) and shortcut read that fp32 map (depth "
"argmax agreement +0.006..0.008 emulated); bf16 = everything bf16",
choices=("bf16", "fp32")),
Knob("BOX_PRECISION", "terms3", "box refiner: terms3 = fp32 activations, three-term convs (PORT_LOG E22); bf16 = "
"single-term convs with BOX_CONV", choices=("terms3", "bf16")),
Knob("BOX_CONV", "HiFi4+fp32+l1acc:w=fp32", "precision of the box refiner's single-term convs "
"(BOX_PRECISION=bf16; ttaw Precision spec)"),
Knob("IMAGE_RULES", "none", "OPT2 P1 (D21, IMAGE_PRECISION=bf16 only): per-module precision of the image "
"branch's single-term convs, 'glob=spec;glob=spec' over the ONNX module names (first "
"match wins; none = IMAGE_CONV everywhere)"),
Knob("PRECISION_RULES", "none", "per-module precision of the BEV trunk / heads' single-term convs (D21): "
"'glob=spec;glob=spec' over the ONNX module names, first match wins (none = the "
"port precision everywhere); fast: FAST_RULES"),
Knob("IMAGE_PRECISION", "terms3", "image branch (ResNet-34, FPN, image heads): terms3 = fp32 activations, every "
"conv as three bf16 terms (needed by the depth / seg2d argmax gates, PORT_LOG.md "
"E20; the accurate profile); bf16 = single-term convs with bf16 activations "
"(the fast profile, D21: judged by the task-level gates; with IMAGE_STEM=fp32 "
"depth / seg2d argmax agreement >= 0.983 / 0.994 on the 45 task-gate frames)",
choices=("terms3", "bf16")),
Knob("CONV_L1", "all", "kxk conv placement (OPT_PLAN #1): none = ttnn's DRAM-sliced path everywhere (with its "
"[1,1,NHW,C] <-> [N,H,W,C] TILE reshape copies); auto = ttnn's L1 path where the input width "
"is not a multiple of 32; all = the L1 path for every kxk conv but the stem (each falls "
"back to DRAM slicing when it does not fit)", choices=("none", "auto", "all")),
Knob("CONV_L1_ABH", "none", "activation block heights retried in order when a conv's L1 path does not fit "
"(none = ttnn's own blocks only)", choices=("none", "64", "32", "64,32", "128,64,32")),
Knob("MM_CFG", "grid", "few-core matmuls (OPT_PLAN #6): stock = as ported; grid = the attention-pool matmuls as "
"2D views, the BEV mean as a batched split-K, Resize2d's W pass as one 2D matmul where "
"C % 32 == 0 (full grid)", choices=("stock", "grid")),
Knob("HOST_PRE", "rows", "host input path (OPT_PLAN #10): rows = the resized cameras stay HWC and upload as a "
"reshape (no CHW round trip); chw = the port's CHW feed + NCHW -> NHWC transpose",
choices=("rows", "chw")),
Knob("HOST_POST", "sparse", "host decode (OPT_PLAN #9): vec = the 2D / 3D peak decode vectorised over classes and "
"cameras and the seg fusion on planar state (identical results); stock = the port's "
"per-plane loops; sparse (OPT round 4 R4.4) = vec with the peak decode evaluated "
"only past a logit pre-filter (identical candidates)",
choices=("stock", "vec", "sparse")),
Knob("SEG_FUSE", "fast", "BEV seg fusion on the host (OPT round 5 R5.2; with HOST_POST vec / sparse): inplace = "
"the planar fusion's float32 ops in the same order into preallocated buffers, row "
"bands on a thread pool (METEOR_SEG_THREADS, default 4), the fused class as the "
"strict > scan (identical results); fast (OPT2 round 6 R6.4) = inplace with the scan as "
"running maxima, no masked copies (identical results); planar = the round-4 path",
choices=("inplace", "planar", "fast")),
Knob("SEG_LSM", "full", "OPT2 round 6 R6.3 (D21, fast profile; SEG_FUSE=inplace / fast): skip = the BEV seg fusion "
"accumulates the raw lane logits (the per-pixel log-softmax shift is class-independent "
"and cancels in the fused argmax in exact arithmetic; float32 near-ties can differ); "
"full = the 9-class log-softmax of render.cpp", choices=("full", "skip")),
Knob("TERMS_EPI", "fused", "three-term conv epilogue (OPT_PLAN #2, bit-identical): stock = the port's fp32 adds "
"and 4-program term splits; epi = one generic_op per conv epilogue (term sum + ReLU) "
"and per split; fused = the producing epilogue also writes the consumer's bf16 terms "
"and the ResNet residual add + ReLU", choices=("stock", "epi", "fused")),
Knob("TERMS_ADD", "split", "fp32 adds next to three-term convs (OPT round 2 R2.2, bit-identical): split = an add "
"feeding only a TermsConv writes that conv's bf16 terms in one program, the refiner "
"stem bias + ReLU and the ctx + paint sum join the conv epilogue; stock = ttnn.add",
choices=("stock", "split")),
Knob("MEAN_CFG", "bmm", "row mean of the BEV / ego maps (OPT round 2 R2.3): bmm = the split-K groups one per core "
"on the grid; auto = ttnn's program config (the groups in sequence on 3 cores)",
choices=("auto", "bmm")),
Knob("CONV_RM", "io", "ROW_MAJOR hand-offs around the DRAM-sliced convs (OPT round 3 R3.1, bit-identical): "
"none = TILE maps (ttnn's [1,1,NHW,C] <-> [N,H,W,C] reshape copies); in = such a conv "
"takes its input untilized; io = it also writes ROW_MAJOR, kept for a conv consumer and "
"tilized for any other (64-128 channel convs; not the lane decoder / ego stem)",
choices=("none", "in", "io")),
Knob("STEM_POOL", "kernel", "the stem's exact fp32 3x3 / s2 max pool (OPT round 3 R3.2, bit-identical): kernel "
"= one generic_op (tt.pool); stock = the 17-program chain of strided slices and "
"maxima",
choices=("stock", "kernel")),
Knob("IMG_IN", "rows", "camera input layout (OPT round 3 R3.3, bit-identical): rows = whole uint8 image rows "
"(2,304 B pages) expanded on the device to the stem's 8-channel bf16 input by one "
"generic_op (tt.kin); px = the port's 3-byte pixel pages + typecast",
choices=("px", "rows")),
Knob("TERMS_ST2", "exact", "OPT round 4 R4.1: a TermsConv's two hi-input products as ONE conv over [W_hi ; W_lo] "
"stacked on the output channels, the epilogue kernel reading the two halves: exact = "
"stacked except the convs where that is not bit-identical (ST2_EXCLUDE); all = "
"everywhere; off = three convs; check = probe (both forms on the warm-ups)",
choices=("off", "exact", "all", "check")),
Knob("PACK", "fast", "OPT round 4 R4.2: the frame's outputs packed into the readback rows by one data-movement "
"generic_op (kernel; tt.pack, bit-identical outputs) or the stock untilize / pad / "
"reshape / concat / slice chain (stock); nchw = the kernel also writes the wide outputs "
"channel-major, so the host conversion is a view (R4.3); fast = nchw with the face reorder "
"rewritten (OPT2 round 4 R4.3, the same bytes)",
choices=("stock", "kernel", "nchw", "fast")),
Knob("REROW", "kernel", "OPT round 5 R5.1: Resize2d's ROW_MAJOR page re-cuts as ttaw's K12 rerow page copy "
"(one data-movement generic_op, bit-identical) or ttnn's RM reshape (stock)",
choices=("stock", "kernel")),
Knob("ARGMAX", "fast", "OPT round 5 R5.3: the seg2d / depth / lane channel argmaxes as one data-movement "
"generic_op (tt.argmax, ArgMaxNC's compare, bit-identical) or the stock transpose + "
"TILE reshape + ArgMaxNC (stock); fast = the same kernel with the per-row reduction "
"rewritten (OPT2 round 4 R4.2, bit-identical)", choices=("stock", "kernel", "fast")),
Knob("READ_JOIN", "views", "OPT round 5 R5.4: views = the packed outputs placed so that every entry that fits a "
"16 MB readback segment stays inside one, the host viewing them in place (only "
"occ is joined); concat = one concat of the segments", choices=("concat", "views")),
Knob("CONV_T4", "refine", "OPT round 6: 4-D TILE hand-offs between ttnn's DRAM-sliced convs with TILE inputs (the "
"lane decoder; bit-identical): io (R6.1) = the producer writes ROW_MAJOR with explicit "
"height slices and its [N,H,W,C] view is tilized, so the consumer's input reshape is a "
"no-op; sums (R6.2) = also the decoder's skip / up-sampled sums and the stationary pool's "
"input stay 4-D; refine (R6.4) = also the seg refiner's stem output (+ range bias, "
"ReLU, bf16 cast) and last skip sum; none = flat TILE hand-offs (two TILE reshape "
"copies each)",
choices=("none", "io", "sums", "refine")),
Knob("WPASS", "grid", "OPT round 6 R6.3 (with MM_CFG=grid): Resize2d's W pass at C % 32 != 0 as the batched "
"matmul with ttaw's fused-batch grid program config (grid; bit-identical, 16 -> 120 cores) "
"or ttnn's own config (flat)", choices=("flat", "grid")),
Knob("TERMS_T4", "t4", "OPT round 7: the det stem's TermsConvs (bit-identical): epi (R7.1) = the epilogue "
"kernel also takes tile-padded parts (the 200x125 maps) and each input term is untilized "
"only for its ROW_MAJOR-input convs; rmo (R7.2) = also the DRAM-path products at widths "
"% 32 != 0 written ROW_MAJOR (explicit height slices) and tilized flat, not reshaped; t4 "
"(R7.3) = also 4-D TILE hand-offs inside the d8 block; none = round 6",
choices=("none", "epi", "rmo", "t4")),
Knob("LIFT", "mk3", "OPT MK3: the camera -> BEV lift as ONE generic_op (tt.lift_mk: per-rig ELL tables, fp32 "
"bilinear weights, not bit-identical by design; a rig with a BEV cell seen by more than 4 "
"cameras is refused with a 400) or the stock 61-program grid_sample lift (stock; any rig)",
choices=("stock", "mk3")),
Knob("RESIZE", "matmul", "OPT2 round 2 (D21, fast profile): every bilinear resize as ONE "
"block-sparse matmul generic_op (kernel; tt.bresize: kron(A_h, A_w) in 32 x 32 blocks, "
"bf16 weight terms, fp32 accumulation, one bf16 rounding; closer to the fp32 reference "
"than the stock path, not bit-identical; an fp32 input (det stem, box refiner) as its two bf16 terms, "
"R2.2) or the stock separable fp32 matmuls (matmul)",
choices=("matmul", "kernel")),
Knob("STEM", "stock", "OPT2 round 3 R3.1 (D21, fast profile; bf16 image branch with IMG_IN=rows): fold = the "
"camera rows written as the space-to-depth form of the zero-padded image (16-channel "
"pixels) and the 7x7 / s2 stem as a 4x4 / s1 conv with re-indexed weights (the same "
"products in another order, not bit-identical); stock = the 8-channel pixels and the "
"7x7 / s2 conv", choices=("stock", "fold")),
Knob("TGATE", "stock", "OPT2 round 3 R3.2 (D21, fast profile): kernel = the temporal fuse's gate glue (slices, "
"softmax, gate scale, t * gate + b, ReLU, bf16 cast) as one generic_op over the merged conv "
"with its gate rows as logit differences (tt.tgate; TF32 operands, not bit-identical); "
"stock = the nine stock programs; mm = (MK-TF) the merged conv as the kernel's first "
"phase (one program from the bf16 raw map to t0); mm4 = mm on the raw map's 4-D TILE copy, "
"writing the 4-D TILE t0 the temporal fuse block slices as is",
choices=("stock", "kernel", "mm", "mm4")),
Knob("BEV_SHARE", "t4", "OPT2 round 3 R3.3 (bit-identical): the BEV maps' layout copies made once and shared: "
"rm = one ROW_MAJOR copy of raw (lane_branch, det stem, occupancy crop) and of fused "
"(traj stem, risk crop, ego attention pool); rm4 = also one 4-D TILE copy of fused for the "
"ego stem's first conv and the pool's keys; t4 = also (OPT2 round 4 R4.1) the raw "
"map's 4-D TILE copy tilized from its ROW_MAJOR copy, and lane_branch.0, the det stem and "
"the traj stem slicing the 4-D TILE copies instead of the ROW_MAJOR ones; off = a copy "
"per consumer",
choices=("off", "rm", "rm4", "t4")),
Knob("DPOOL", "kernel", "OPT2 round 4 R4.4 (bit-identical): kernel = the depth head's exact fp32 avgpool2 "
"(depth_mean) from four quadrant maps written by one data-movement generic_op (tt.dpool) "
"and the same fp32 adds in the same order; stock = the untilize / slice / tilize / reshape "
"chain", choices=("stock", "kernel")),
Knob("DET_STEM", "terms", "OPT2 round 5 (D21, fast profile): single = the det stem's convs (stem / d8 Down, u, m) "
"single-term with bf16 activations and the port's fp32 weights at HiFi4 (PRECISION_RULES "
"may change them), the merged hm | reg head three-term on det_m's fp32 output; single_bf "
"= det_m writes bf16; terms = the three-term det stem (with TERMS_HI)",
choices=("terms", "single", "single_bf")),
Knob("BRES", "cache", "OPT2 round 5 (RESIZE=kernel, bit-identical): cache = the BRES resize kernel's reader keeps "
"the input blocks and weight tiles in L1 caches (kernels/bres_reader2.cpp: each input tile "
"read from DRAM about once per core instead of ~10 times); v1 = every entry read from DRAM",
choices=("v1", "cache")),
Knob("READ_VIEW", "view", "OPT2 round 5 (host, identical outputs): view = the readback segments copied into fresh "
"host tensors and viewed zero-copy (ttaw TraceRunner read_view); copy = copied into "
"preallocated host tensors and converted by to_numpy", choices=("copy", "view")),
Knob("DET_D8", "tile", "OPT2 round 5 R5.4 (DET_STEM=single*, bit-identical): tile = the det stem's d8 block keeps "
"TILE hand-offs (its stride-2 conv sliced a ROW_MAJOR input in 1.5 ms PaddedSlices); rm = "
"CONV_RM's ROW_MAJOR hand-offs", choices=("rm", "tile")),
Knob("STEM_IN", "t4", "OPT2 round 6 R6.1 (STEM=fold, bit-identical): t4 = the folded stem input written as a 4-D "
"TILE tensor (the stem conv's DRAM-path slices read whole tiles); rm = the flat ROW_MAJOR "
"form of 32 B pages", choices=("rm", "t4")),
Knob("ATTN_IN", "bf16", "OPT2 round 6 R6.2 (bit-identical): bf16 = the attention pools' W-pool matmul reads the "
"bf16 map (fp32 output); fp32 = the map typecast to fp32 first", choices=("fp32", "bf16")),
Knob("EPI_ADD", "fused", "OPT2 round 6 R6.5 (bit-identical): fused = bf16(relu(a + b)) of the bf16 seg refiner's "
"stem and the fast image branch's first block as one add with a ReLU post-activation "
"and a bf16 output; stock = add, relu, typecast", choices=("stock", "fused")),
Knob("REF_TAIL", "fused", "OPT2 round 6 R6.6 (bit-identical): fused = the refiners' residual tails c + s tanh(res / "
"s) as two programs (activations fused into the multiply and the add); stock = four",
choices=("stock", "fused")),
Knob("READ_OCC", "lazy", "OPT2 round 6 R6.7 (identical outputs): lazy = the 25.6 MB occupancy logits (dense heads "
"only) stay on the device until read; eager = the whole readback after every replay",
choices=("eager", "lazy")),
Knob("RAW4", "bres", "OPT2 MK-TF follow-up (layout only): bres = the raw BEV map's 4-D TILE form written by a "
"second call of the lift's BRES resize kernel (RESIZE=kernel) instead of tilized from its "
"ROW_MAJOR copy; copy = the tilize", choices=("copy", "bres")),
Knob("FUSED_OUT", "kernel", "OPT2 MK-FO (bit-identical; with TGATE=mm4): kernel = the fused BEV map (raw + t) and "
"its shared ROW_MAJOR / 4-D TILE copies written by one generic_op from the temporal "
"fuse block's ROW_MAJOR output; stock = tilize, add, untilize and 4-D tilize",
choices=("stock", "kernel")),
Knob("KTILE", "kernel", "OPT2 (bit-identical): kernel = the conv hand-offs' ROW_MAJOR -> TILE conversions (flat or "
"4-D, bf16 / fp32, >= 4096 pixels) as one data-movement generic_op (same-bank page runs, "
"local NOC copies) and the risk crop tilized straight from the ROW_MAJOR map; tile = the "
"conversions only; stock = ttnn.to_layout", choices=("stock", "tile", "kernel")),
Knob("T4_CHAIN", "on", "OPT2 (layout only, with KTILE): on = DRAM-sliced convs hand a 4-D TILE map (their ROW_MAJOR "
"output tilized by KTILE) to the consumers that slice it as is (temporal fuse block, "
"lane_branch.3, risk and traj stem blocks; the risk crop 4-D); off = ROW_MAJOR hand-offs",
choices=("off", "on")),
Knob("KUNTILE", "kernel", "OPT2 (bit-identical): kernel = the TILE -> ROW_MAJOR conversions of the conv hand-offs "
"and the raw map's shared ROW_MAJOR copy as one data-movement generic_op; stock = "
"ttnn.to_layout", choices=("stock", "kernel")),
Knob("TERMS_HI", "none", "OPT2 round 3 R3.4 (D21, fast profile: det_stem/*): comma-separated globs of the "
"three-term convs (det path, box refiner) that drop their x_lo * W_hi product (bf16 "
"activations times ~16-bit weights); none = every TermsConv keeps three products"),
Knob("RESIZE_SKIP", "none", "RESIZE=kernel only: comma-separated globs of resize names (lift.resize, "
"fpn.resize*, seg.resize_u*, dec.resize_u*, refiner/seg.resize*) that keep the stock "
"matmuls (none = the kernel wherever the input is bf16)"),
])
# D21 serve profiles: the defaults a profile gives the knobs it governs (a knob set in the environment or as a compile
# param keeps its value). accurate = every knob's own default (the release).
FAST_PRECISION = "HiFi2+fp32+l1acc:w=bfloat16"
PORT_PRECISION = "HiFi4+fp32+l1acc:w=fp32"
# OPT_PLAN A4: the BEV trunk's bf16-activation convs (lane decoder, lane branch, temporal fuse block, occupancy stem,
# risk trunk, seg refiner) with bf16 weights at HiFi2; the output layers, the temporal gate, the planner (ego / sem /
# traj), the stationary head and the det path keep the port precision (upstream's dense-exception list)
FAST_RULES = ";".join(["refiner/seg/out/out.3=" + PORT_PRECISION] + [
f"{g}={FAST_PRECISION}" for g in ("dec/*", "lane_branch/*", "tfuse3_slim/*", "occ_stem/*", "risk_head/risk_head.0/*",
"risk_head/risk_head.1/*", "refiner/seg/*")])
# OPT2 P1 (D21): the image heads that the emulation sweep (precision_emulation.py wsweep) and the device task gates
# found weight-tolerant: the det2d pyramid + merged heads, the TL head and the seg2d head with bf16 weights at HiFi2.
# The encoder (stem, ResNet layers, FPN laterals / fuse), the depth head and ctx + paint keep the port's fp32 weights
# (layer4 / FPN / ctx: depth min 0.965-0.980 or box recall 0.956 on the device)
FAST_IMAGE_RULES = ";".join(f"{g}={FAST_PRECISION}" for g in ("det2d_*", "hm2d_head*", "tl_head/*", "seg_head/*"))
PROFILE_DEFAULTS: Dict[str, Dict[str, str]] = {
"accurate": {},
"fast": {"IMAGE_PRECISION": "bf16", "IMAGE_CONV": PORT_PRECISION, "IMAGE_STEM": "fp32",
"BOX_PRECISION": "bf16", "BOX_CONV": PORT_PRECISION, "PRECISION_RULES": FAST_RULES,
"RESIZE": "kernel", "STEM": "fold", "TGATE": "mm4",
"TERMS_HI": "det_stem/*", "DET_STEM": "single_bf", "SEG_LSM": "skip", "IMAGE_RULES": FAST_IMAGE_RULES},
}
def apply_profile(knobs: Any) -> Any:
"""Give the knobs that ``knobs.PROFILE`` governs its defaults, unless set explicitly (source != default)."""
from .ttaw.knobs import KnobValues
values, sources = knobs.as_dict(), {k: knobs.source(k) for k in knobs}
for name, value in PROFILE_DEFAULTS[str(knobs.PROFILE)].items():
if sources[name] == "default":
values[name], sources[name] = KNOBS.knobs[name].parse(value), "profile"
return KnobValues(KNOBS.prefix, values, sources)
def tt_model_kwargs(knobs: Any) -> Dict[str, str]:
"""The ``TtMETEOR`` keyword arguments that the load-time knobs choose (``apply_profile`` applied first by the caller):
the API and ``scripts/stress_frames.py`` build the device graph from the same mapping."""
return {
"image_precision": str(knobs.IMAGE_PRECISION),
"conv_l1": str(knobs.CONV_L1),
"mm_cfg": str(knobs.MM_CFG),
"conv_l1_abh": str(knobs.CONV_L1_ABH),
"terms_epi": str(knobs.TERMS_EPI),
"terms_add": str(knobs.TERMS_ADD),
"mean_cfg": str(knobs.MEAN_CFG),
"conv_rm": str(knobs.CONV_RM),
"stem_pool": str(knobs.STEM_POOL),
"img_in": str(knobs.IMG_IN),
"terms_st2": str(knobs.TERMS_ST2),
"pack": str(knobs.PACK),
"rerow": str(knobs.REROW),
"argmax": str(knobs.ARGMAX),
"read_join": str(knobs.READ_JOIN),
"conv_t4": str(knobs.CONV_T4),
"wpass": str(knobs.WPASS),
"terms_t4": str(knobs.TERMS_T4),
"lift": str(knobs.LIFT),
"image_conv": str(knobs.IMAGE_CONV),
"box_precision": str(knobs.BOX_PRECISION),
"box_conv": str(knobs.BOX_CONV),
"precision_rules": str(knobs.PRECISION_RULES),
"image_stem": str(knobs.IMAGE_STEM),
"image_rules": str(knobs.IMAGE_RULES),
"resize": str(knobs.RESIZE),
"resize_skip": str(knobs.RESIZE_SKIP),
"stem": str(knobs.STEM),
"tgate": str(knobs.TGATE),
"bev_share": str(knobs.BEV_SHARE),
"terms_hi": str(knobs.TERMS_HI),
"dpool": str(knobs.DPOOL),
"det_stem": str(knobs.DET_STEM),
"bres": str(knobs.BRES),
"read_view": str(knobs.READ_VIEW),
"det_d8": str(knobs.DET_D8),
"stem_in": str(knobs.STEM_IN),
"attn_in": str(knobs.ATTN_IN),
"epi_add": str(knobs.EPI_ADD),
"ref_tail": str(knobs.REF_TAIL),
"read_occ": str(knobs.READ_OCC),
"raw4": str(knobs.RAW4),
"fused_out": str(knobs.FUSED_OUT),
"ktile": str(knobs.KTILE),
"t4_chain": str(knobs.T4_CHAIN),
"kuntile": str(knobs.KUNTILE),
}
def load_sample(path: Any, *, with_stream: bool = True) -> Dict[str, Any]:
"""``model(**load_sample("samples/<name>.json"))``: cameras + calibration preset + ego speed (+ stream)."""
from .host.inputs import load_sample as _load
return _load(path, with_stream=with_stream)
class METEOR(ModelBase):
"""METEOR (TIER IV, AutowareFoundation/meteor) on one Blackhole p150. Create it with :meth:`from_pretrained`."""
MODEL_NAME = "meteor-p150"
ENV_PREFIX = "METEOR" # prefix of the environment knobs (SERVING.md section 3.4)
DEFAULT_REPO = "AutowareFoundation/meteor"
DEFAULT_TAG = "v1.0" # the AutowareFoundation release tag (no Autoware ansible pin exists for METEOR)
# the commit DEFAULT_TAG points to (pinned: tags can move)
DEFAULT_REVISION = "01a5f6d71df5ecbbb5853ec600825481d57b9c6b"
ALLOW_PATTERNS = ["meteor_v157c3Z.onnx", "meteor_v157.param.yaml", "LICENSE", "SHA256SUMS"]
VARIANTS = ["default"] # load-time: selects weights files and trace shapes
DEFAULT_VARIANT = "default"
INPUT_KIND = "multicam" # lidar | camera | multicam | lidar+multicam | planner
CAMERA_ORDER = CAMERAS # the network's fixed camera order (meteor_v157.param.yaml:10)
POINT_FIELDS = tio.DEFAULT_POINT_FIELDS # unused (camera model); kept for the shared /info schema
LABELS = list(_LABELS) # 3D box classes (label_id 0 vehicle, 1 VRU)
# Per-request knobs: name -> (type, min, max, default). Host-side post-processing only (SPEC section 9, RT-host;
# render.cpp:345-376 defaults).
RUNTIME_PARAMS = {
"det3d_threshold": (float, 0.0, 1.0, 0.15),
"det3d_topk": (int, 1, 1024, 64),
"vehicle_threshold": (float, 0.0, 1.0, 0.35),
"vru_threshold": (float, 0.0, 1.0, 0.15),
"bev_nms_iou": (float, 0.0, 1.0, 0.3),
"bev_nms_containment": (float, 0.0, 1.0, 0.6),
"stationary_logit_threshold": (float, None, None, 0.0),
"det2d_threshold": (float, 0.0, 1.0, 0.30),
"det2d_topk": (int, 1, 1024, 48),
"det2d_hide": (str, None, None, "7"),
"unk2d": (bool, None, None, True),
"unk2d_threshold": (float, 0.0, 1.0, None),
"ground_z": (float, -5.0, 5.0, 0.0),
"mode_hysteresis": (float, 0.0, 10.0, 0.35),
"straight_margin": (float, 0.0, 10.0, 1.0),
"seg_fuse": (bool, None, None, True),
"thin_road_edge": (bool, None, None, True),
"yaw_smoothing": (bool, None, None, True),
"heads": (bool, None, None, False),
}
EXTRA_INPUTS = ("ego_speed",)
DEVICE_DEFAULTS = DEVICE_DEFAULTS # validated open parameters (device.py)
# ---- port-specific hooks (called by ModelBase; keep host work out of _forward) ---------------------------
def _build(self) -> None:
"""Weights (``reference.weights.MeteorWeights``: the ONNX parameters by consuming node) -> the device graph of
``tt_meteor.tt`` (``TtMETEOR``: the whole graph as ONE ``TraceRunner`` variant ``frame``, no capture yet)."""
self._build_host()
from .host.calib import CalibrationCache
from .reference.weights import MeteorWeights
from .tt.model import TtMETEOR
from .tt.params import MeteorParams
params = MeteorParams(MeteorWeights(self.weights_path), self.cfg)
self.calib_cache = CalibrationCache(points=params.ground_points())
self.tt = TtMETEOR(self.device, params, num_command_queues=self.device_info.get("num_command_queues"),
**tt_model_kwargs(self.knobs))
self.runner = self.tt.runner
def _warm_one(self, variant: Any) -> None:
"""Warm-up of the ``frame`` variant (compiles every program) and its capture."""
self.runner.capture()
def _build_host(self) -> None:
"""The host half of the model: load-time knobs and the per-stream host temporal states (no device, no
weights; the host tests call it directly)."""
from .host.postprocess import PostConfig
from .reference.config import default_config
params = {k.upper(): v for k, v in getattr(self, "compile_params", {}).items() if k.upper() in KNOBS.knobs}
self.knobs = apply_profile(KNOBS.read(**params))
self.cfg = default_config(input_norm=str(self.knobs.INPUT_NORM),
depth_mean_bins=str(self.knobs.DEPTH_MEAN_BINS))
self.post = PostConfig()
from .host.postprocess import set_host_post
set_host_post(str(self.knobs.HOST_POST))
from .host.temporal import set_seg_fuse
set_seg_fuse(str(self.knobs.SEG_FUSE))
from .host.temporal import set_seg_lsm
set_seg_lsm(str(self.knobs.SEG_LSM))
self._streams: "OrderedDict[str, Any]" = OrderedDict()
def _stream_state(self, stream: Optional[Mapping[str, Any]]):
from .host.temporal import StreamState
sid = str((stream or {}).get("id", "default"))
st = self._streams.get(sid)
if st is None:
st = self._streams[sid] = StreamState()
while len(self._streams) > int(self.knobs.MAX_STREAMS):
self._streams.popitem(last=False)
elif (stream or {}).get("reset"):
st.reset()
self._streams.move_to_end(sid)
return sid, st
def _prepare(self, points: Any = None, *, images: Any = None, calibration: Any = None, stream: Any = None,
ego_speed: Any = None, **_: Any) -> Dict[str, Any]:
"""Cameras + calibration + ego speed -> the graph feed (``host.inputs.prepare_request``)."""
from .host.inputs import prepare_request
if points is not None:
raise tio.InputError("this model takes eight camera images (+ calibration, ego_speed), not a point cloud")
frame = prepare_request(images, calibration, ego_speed, stream, layout=str(self.knobs.HOST_PRE))
sid, state = self._stream_state(stream)
return {"frame": frame, "stream_id": sid, "state": state}
def _forward(self, prepared: Dict[str, Any]) -> Any:
"""Upload (cameras, v0; the lift tables when the rig changed) + replay of the ``frame`` trace + one packed
read -> the 19 outputs (``tt.unpack``: ONNX names / layouts / dtypes)."""
frame = prepared["frame"]
geom = self.calib_cache.get(frame.K, frame.T_cam_ego)
return self.tt.run_frame(frame, geom)
def _postprocess(self, raw: Any, prepared: Dict[str, Any], params: Dict[str, Any]) -> Output:
"""METEOR's C++ host decode on the 19 outputs (``host.result.build_output``) with this stream's state."""
from .host.result import build_output
hide = tuple(int(v) for v in str(params.get("det2d_hide", "7")).replace(" ", "").split(",") if v != "")
knobs = {k: v for k, v in params.items() if k not in ("det2d_hide", "heads")}
cfg = self.post.with_params(det2d_hide=hide, **knobs)
return build_output(raw, prepared["frame"], cfg, prepared["state"], heads=bool(params.get("heads")),
model=self.MODEL_NAME, meta={"stream_id": prepared["stream_id"]})
def _release(self) -> None:
"""Release the traces and persistent device tensors; also called when ``from_pretrained`` fails half-way."""
runner = getattr(self, "runner", None)
if runner is not None:
runner.release()
def extra_info(self) -> Dict[str, Any]:
runner = getattr(self, "runner", None)
info: Dict[str, Any] = {"knobs": self.knobs.as_dict()} if hasattr(self, "knobs") else {}
if runner is not None:
info["trace"] = runner.describe()
return info
|