code/scripts/make_demo.py

#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
"""Demo media of the model card from the p150 outputs, and their agreement with the fp32 CPU reference (workspace
tool: besides the shipped samples it reads the public-dataset frames and CPU runs of
``research/sceneseg/public_data``, which do not ship).

Two steps, so the device is held only for inference:

    # 1. device (tt-env, under bin/devrun): the production API (SceneSeg.from_pretrained, then model(rgb array or
    #    path)), both channel orders, on the shipped samples (9 tutorial PNGs, 6 comma10k images with the training
    #    ROI crop) and the demo frames (PandaSet front camera, nuScenes CAM_FRONT, KITTI 2:1 crops; PandaSet 019
    #    frames 00-79 in bgr only, the order of the CPU runs)
    bin/devrun -t 1200 -- python code/scripts/make_demo.py run --out logs/sceneseg/m5_docs/demo
    # 2. CPU (research venv: OpenCV + PIL): the renders into media/ (the research renderer's overlay, palette and
    #    caption bars: research/sceneseg/public_data/scripts/ss_render.py)
    tools/research-venv/bin/python -I code/scripts/make_demo.py render --out logs/sceneseg/m5_docs/demo --media media
    # 3. device, optional: the served trace's 320x640 class map of EVERY CPU run of research/sceneseg/public_data/runs
    #    (952 runs: KITTI, comma10k val, nuScenes, PandaSet, both orders where the CPU ran both), written with the
    #    research runs' file names, so the research metrics (ss_sanity.py --cls-dir) score the p150 like the CPU
    bin/devrun -t 1800 -- python code/scripts/make_demo.py cls --out logs/sceneseg/m5_docs/demo

``run`` writes ``<out>/run/<frame>_<variant>.npz`` (``class_map``: the API's class map at the source size, i.e. what
``/predict`` returns; the published mask is ``class_map == 1``) and ``<out>/agreement.json``: per frame the
agreement of the published mask and of the class map with the CPU reference's, both at the source size (the CPU
320x640 class map resized with the node's INTER_NEAREST), and ``timing_ms``. The frozen device gates
(``tests/test_e2e_device.py``) compare at 320x640 and also gate the logits.
"""
from __future__ import annotations

import argparse
import json
import sys
import time
from pathlib import Path

import numpy as np

HERE = Path(__file__).resolve()
CODE = HERE.parents[1]
sys.path.insert(0, str(CODE))

WS = Path("/home/ubuntu/experiments/tt-models")
PUB = WS / "research" / "sceneseg" / "public_data"
SAMPLES = CODE / "tt_sceneseg" / "samples"
SMALL_GOLDENS = CODE / "tt_sceneseg" / "tests" / "goldens"
VARIANTS = ("bgr", "rgb")
TUTORIAL = ("highway_normal_1", "highway_normal_2", "highway_normal_3", "highway_normal_4", "highway_normal_5",
            "highway_rain_1", "highway_rain_2", "highway_rain_3", "highway_snow")
GOLDEN_IDS = ("ps019_f40_front_camera", "ps005_f40_front_camera", "ps029_f40_front_camera", "ps090_f40_front_camera",
              "ps056_f40_front_camera", "ps065_f40_front_camera", "ns0103_k28_CAM_FRONT")
KITTI_PICKS = PUB / "logs" / "render_kitti_picks.json"


# ------------------------------------------------------------------------------------------------ frame registry
def _cv2():
    import cv2

    return cv2


def _sha256_array(a: np.ndarray) -> str:
    import hashlib

    a = np.ascontiguousarray(a)
    h = hashlib.sha256()
    h.update(f"{a.dtype.str}{a.shape}".encode())
    h.update(a.tobytes())
    return h.hexdigest()


def _crop_view(img: np.ndarray, crop) -> np.ndarray:
    if crop in (None, "", "full"):
        return img
    h, w = img.shape[:2]
    if crop == "c2to1":
        cw = min(w, 2 * h)
        x0 = (w - cw) // 2
        return img[:, x0:x0 + cw]
    if crop == "roi70":
        return img[:int(h * 0.7), :w - 1]
    raise ValueError(crop)


def _runs(set_name: str):
    out = {}
    for line in (PUB / "runs" / f"{set_name}.jsonl").read_text().splitlines():
        r = json.loads(line)
        out.setdefault(r["id"], {**r, "variants": set()})["variants"].add(r["variant"])
    return out


def frames():
    """``[{id, set, variants, load(): BGR uint8 view, cpu(variant): uint8 320x640 class map, path?}]``."""
    from tt_sceneseg.tests.sceneseg_eval import COMMA10K_GOLDENS, comma10k_samples, load_rgb, roi70

    cv2 = _cv2()
    out = []
    for stem in TUTORIAL:                      # shipped, Apache-2.0: passed as the PNG path, like a user would
        path = SAMPLES / f"{stem}.png"
        out.append({"id": f"tut_{stem}", "set": "tutorial", "variants": VARIANTS, "path": path,
                    "load": (lambda p=path: np.ascontiguousarray(load_rgb(p)[:, :, ::-1])),
                    "cpu": (lambda v, s=stem: np.load(SMALL_GOLDENS / f"{s}.npz")[f"{v}.class_map"])})
    for name, (img, _) in comma10k_samples().items():     # shipped, MIT: the training ROI crop
        out.append({"id": f"c10k_{name[:4]}", "set": "comma10k", "variants": VARIANTS,
                    "load": (lambda p=img: np.ascontiguousarray(roi70(load_rgb(p)[:, :, ::-1]))),
                    "cpu": (lambda v, n=name[:4]: np.load(COMMA10K_GOLDENS / f"{n}.npz")[f"cls_{v}"])})
    for fid in GOLDEN_IDS:                     # not shipped: the converted public frames (source_bgr.png = bgr8)
        out.append({"id": fid, "set": fid[:2], "variants": VARIANTS,
                    "load": (lambda f=fid: cv2.imread(str(PUB / "inputs" / f / "source_bgr.png"), cv2.IMREAD_COLOR)),
                    "cpu": (lambda v, f=fid: np.load(PUB / "golden" / f / "ort_fp32.npz")[f"cls_{v}"])})
    picks = json.loads(KITTI_PICKS.read_text())["kitti_frames"] if KITTI_PICKS.is_file() else []
    runs = {**_runs("demo019"), **{k: v for k, v in _runs("kitti").items()
                                   if any(k == f"kitti_{p}_c2to1" for p in picks)}}
    for fid, r in sorted(runs.items()):        # not shipped: decoded from the dataset files like the research runs
        if fid in GOLDEN_IDS:                  # ps019_f40 is also a converted golden frame (registered above)
            continue

        def load(r=r):
            full = cv2.imdecode(np.fromfile(str(WS / r["source_rel"]), dtype=np.uint8), cv2.IMREAD_COLOR)
            if _sha256_array(full) != r["decoded_sha256"]:
                raise RuntimeError(f"{r['id']}: decoded pixels differ from the research run")
            return np.ascontiguousarray(_crop_view(full, r["crop"]))

        out.append({"id": fid, "set": r["set"], "variants": tuple(v for v in VARIANTS if v in r["variants"]),
                    "load": load,
                    "cpu": (lambda v, f=fid: cv2.imread(str(PUB / "runs" / "cls" / f"{f}_{v}.png"),
                                                        cv2.IMREAD_UNCHANGED))})
    return out


def cpu_at_source(cls320: np.ndarray, hw) -> np.ndarray:
    """The CPU class map at the source size, resized like the node's post-processing (INTER_NEAREST)."""
    cv2 = _cv2()
    return cv2.resize(cls320, (int(hw[1]), int(hw[0])), interpolation=cv2.INTER_NEAREST)


# ----------------------------------------------------------------------------------------------------- 1. run
def run(out: Path) -> None:
    from tt_sceneseg import SceneSeg

    (out / "run").mkdir(parents=True, exist_ok=True)
    reg = frames()
    rows, meta = {}, {}
    for variant in VARIANTS:
        t0 = time.perf_counter()
        with SceneSeg.from_pretrained(variant=variant) as model:     # the published path: pinned Hub weights
            meta[variant] = {"load_s": round(time.perf_counter() - t0, 1), "weights": model.info["weights"],
                             "device": model.device_info}
            print(variant, json.dumps(meta[variant], default=str), flush=True)
            for f in reg:
                if variant not in f["variants"]:
                    continue
                res = model(str(f["path"])) if "path" in f else model(f["load"]()[:, :, ::-1])   # RGB in
                cmap = np.asarray(res.class_map, np.uint8)
                np.savez_compressed(out / "run" / f"{f['id']}_{variant}.npz", class_map=cmap)
                cpu = cpu_at_source(f["cpu"](variant), cmap.shape)
                rows[f"{f['id']}_{variant}"] = {
                    "id": f["id"], "set": f["set"], "variant": variant, "source_hw": list(cmap.shape),
                    "mask_agreement": round(float(((cmap == 1) == (cpu == 1)).mean()), 6),
                    "class_map_agreement": round(float((cmap == cpu).mean()), 6),
                    "fg_fraction_tt": round(float((cmap == 1).mean()), 5),
                    "fg_fraction_cpu": round(float((cpu == 1).mean()), 5),
                    "timing_ms": {k: round(float(v), 2) for k, v in res.timing_ms.items()}}
            print(variant, "done", flush=True)
    summary = {}
    for (s, v) in sorted({(r["set"], r["variant"]) for r in rows.values()}):
        sel = [r for r in rows.values() if r["set"] == s and r["variant"] == v]
        summary[f"{s}_{v}"] = {"frames": len(sel),
                               "mask_agreement_min": min(r["mask_agreement"] for r in sel),
                               "mask_agreement_mean": round(float(np.mean([r["mask_agreement"] for r in sel])), 6),
                               "class_map_agreement_min": min(r["class_map_agreement"] for r in sel)}
    (out / "agreement.json").write_text(json.dumps({"meta": meta, "summary": summary, "frames": rows},
                                                   indent=1, default=str) + "\n")
    print(json.dumps(summary, indent=1), flush=True)


# ------------------------------------------------------------------------------------------------------- 3. cls
def cls_maps(out: Path) -> None:
    """The served trace (``model.runner``: what ``model()`` uploads, replays and reads back) on every research run
    record, with the record's decode (sha256-checked), view crop and the API's host squash; ``<out>/cls/<id>_<v>.png``
    (uint8 320x640, the research runs' format) and ``<out>/cls_agreement.json`` (argmax agreement with the CPU run)."""
    from tt_sceneseg import SceneSeg
    from tt_sceneseg.host.preprocess import prepare_bgr

    cv2 = _cv2()
    (out / "cls").mkdir(parents=True, exist_ok=True)
    recs = []
    for f in sorted((PUB / "runs").glob("*.jsonl")):
        recs += [json.loads(line) for line in f.read_text().splitlines()]
    agree, meta = {}, {}
    for variant in VARIANTS:
        todo = [r for r in recs if r["variant"] == variant]
        t0 = time.perf_counter()
        with SceneSeg.from_pretrained(variant=variant) as model:
            meta[variant] = {"load_s": round(time.perf_counter() - t0, 1), "runs": len(todo),
                             "weights": model.info["weights"], "device": model.device_info}
            decoded = {}
            for r in todo:
                src = WS / r["source_rel"]
                if src not in decoded:
                    decoded.clear()
                    full = cv2.imdecode(np.fromfile(str(src), dtype=np.uint8), cv2.IMREAD_COLOR)
                    if _sha256_array(full) != r["decoded_sha256"]:
                        raise RuntimeError(f"{r['id']}: decoded pixels differ from the research run")
                    decoded[src] = full
                view = np.ascontiguousarray(_crop_view(decoded[src], r["crop"]))
                cmap = np.asarray(model.runner(prepare_bgr(view, channel_order=variant).device_input())["class_map"],
                                  np.uint8).reshape(320, 640)
                cv2.imwrite(str(out / "cls" / f"{r['id']}_{variant}.png"), cmap)
                ref = cv2.imread(str(PUB / "runs" / "cls" / f"{r['id']}_{variant}.png"), cv2.IMREAD_UNCHANGED)
                agree[f"{r['id']}_{variant}"] = {"set": r["set"], "agreement": round(float((cmap == ref).mean()), 6)}
        print(variant, json.dumps(meta[variant], default=str), flush=True)
    by = {}
    for k, v in agree.items():
        by.setdefault(f"{v['set']}_{k.rsplit('_', 1)[1]}", []).append(v["agreement"])
    summary = {k: {"runs": len(v), "min": min(v), "mean": round(float(np.mean(v)), 6)} for k, v in sorted(by.items())}
    (out / "cls_agreement.json").write_text(json.dumps({"meta": meta, "summary": summary, "runs": agree},
                                                       indent=1, default=str) + "\n")
    print(json.dumps(summary, indent=1), flush=True)


# -------------------------------------------------------------------------------------------------- 2. render
def render(out: Path, media: Path) -> None:
    sys.path.insert(0, str(PUB / "scripts"))
    import ss_render as R                     # research renderer: overlay, palette, caption bars, attribution texts
    from PIL import Image, ImageDraw

    cv2 = _cv2()
    agree = json.loads((out / "agreement.json").read_text())
    rows = agree["frames"]
    reg = {f["id"]: f for f in frames()}
    media.mkdir(parents=True, exist_ok=True)
    TT = "Tenstorrent Blackhole p150 output (sceneseg-p150: ETH dispatch, 12x10 grid, optimized second release)"

    def tt(fid, v):
        return np.load(out / "run" / f"{fid}_{v}.npz")["class_map"]

    def rgb_of(fid):
        return reg[fid]["load"]()[:, :, ::-1].copy()

    def tile(fid, v, w, h, label=None, fs=13):
        img = Image.fromarray(R.overlay(R.resize_rgb(rgb_of(fid), w, h), R.resize_cls(tt(fid, v), w, h)))
        if label:
            R.chip_label(img, (6, 6), label, fs=fs)
        return img

    def ag(fid, v):
        r = rows[f"{fid}_{v}"]
        return f"published mask {100 * r['mask_agreement']:.2f} % equal to the fp32 CPU reference"

    def save(img, name, q=88):
        p = media / name
        if name.endswith(".png"):
            img.save(p, optimize=True)
        else:
            img.save(p, quality=q, optimize=True, progressive=True, subsampling=0)
        print("wrote", p, img.size, p.stat().st_size, flush=True)

    def diff_panel(fid, v, w, h):
        """The image dimmed, with the pixels where the p150 and CPU class maps differ in white (dilated 1 px so
        single pixels stay visible at display size)."""
        a = tt(fid, v)
        b = cpu_at_source(reg[fid]["cpu"](v), a.shape)
        d = (a != b).astype(np.uint8)
        d = cv2.resize(cv2.dilate(d, np.ones((3, 3), np.uint8)), (w, h), interpolation=cv2.INTER_NEAREST) > 0
        base = (R.resize_rgb(rgb_of(fid), w, h).astype(np.float32) * 0.35 + 255 * 0.65 * 0.15).astype(np.uint8)
        base[d] = (255, 255, 255)
        return Image.fromarray(base), int((a != b).sum()), a.size

    def tt_vs_cpu(fid, v, name, title, footer, w=480, h=None):
        src = rgb_of(fid)
        h = h or int(round(w * src.shape[0] / src.shape[1]))
        cpu = cpu_at_source(reg[fid]["cpu"](v), tt(fid, v).shape)
        left = tile(fid, v, w, h, "p150 (TT)")
        right = Image.fromarray(R.overlay(R.resize_rgb(src, w, h), R.resize_cls(cpu, w, h)))
        R.chip_label(right, (6, 6), "fp32 CPU reference (ONNX Runtime)")
        top = Image.new("RGB", (2 * w, h), R.SURFACE)
        top.paste(left, (0, 0))
        top.paste(right, (w, 0))
        ImageDraw.Draw(top).line([(w, 0), (w, h)], fill=R.SURFACE, width=2)
        dp, n, tot = diff_panel(fid, v, 2 * w, 2 * h)
        R.chip_label(dp, (6, 6), f"pixels where the two class maps differ (white): {n} of {tot}")
        r = rows[f"{fid}_{v}"]
        bar = R.caption_bar(2 * w, title, [
            f"{TT}, {v} channel order. Published mask {100 * r['mask_agreement']:.3f} % and 3-class map "
            f"{100 * r['class_map_agreement']:.3f} % of pixels equal to the fp32 CPU reference of the same network "
            f"(same weights and pre-/post-processing), at the source resolution."], footer=footer)
        save(R.stack([top, dp, bar]), name, q=90)

    PS_FOOT = R.PS_FOOT
    NS_FOOT = R.NS_FOOT
    KITTI_FOOT = R.KITTI_FOOT
    TUT_FOOT = ("VisionPilot SceneSeg tutorial image (autowarefoundation/vision_pilot @ ca58cb50, Apache-2.0), shipped "
                "in code/tt_sceneseg/samples/. Changes: model output overlaid.")
    C10K_FOOT = ("comma10k (commaai/comma10k @ 6c205fe, MIT, (c) 2020 Comma.ai, Inc.), shipped in "
                 "code/tt_sceneseg/samples/comma10k/. Changes: training ROI crop (top 70 %), resized, prediction / "
                 "ground truth overlaid.")

    # --- the shipped sample: output, and p150 vs CPU
    fid = "tut_highway_normal_1"
    src = rgb_of(fid)
    h0, w0 = src.shape[:2]
    img = tile(fid, "bgr", w0, h0)
    bar = R.caption_bar(w0, "SceneSeg on the shipped sample (highway_normal_1.png, 727x414) - p150 output",
                        [f"{TT}; deployed BGR order; foreground {R.fgpct(tt(fid, 'bgr')):.1f} % of pixels; "
                         f"{ag(fid, 'bgr')}."], footer=TUT_FOOT, fs=13)
    save(R.stack([img, bar]), "sceneseg_highway_normal_1_tt.jpg", q=92)
    tt_vs_cpu(fid, "bgr", "sceneseg_highway_normal_1_tt_vs_cpu.png",
              "p150 vs fp32 CPU reference on the shipped sample (highway_normal_1.png)", TUT_FOOT, w=480)

    # --- the 9 tutorial images
    grid = Image.new("RGB", (960, 3 * 180), R.SURFACE)
    for i, stem in enumerate(TUTORIAL):
        grid.paste(tile(f"tut_{stem}", "bgr", 320, 180, stem, fs=11), ((i % 3) * 320, (i // 3) * 180))
    worst = min(rows[f"tut_{s}_bgr"]["mask_agreement"] for s in TUTORIAL)
    bar = R.caption_bar(960, "SceneSeg on the 9 shipped tutorial images - p150 output",
                        [f"{TT}; deployed BGR order. Published mask agreement with the fp32 CPU reference "
                         f">= {100 * worst:.2f} % on every image."], footer=TUT_FOOT)
    save(R.stack([grid, bar]), "sceneseg_tutorial_samples_tt.jpg")

    # --- comma10k: p150 vs ground truth
    from tt_sceneseg.tests.sceneseg_eval import COMMA10K_GOLDENS, comma10k_ious

    parts = []
    for n in ("0009", "0019", "0049"):
        fid = f"c10k_{n}"
        gt = np.load(COMMA10K_GOLDENS / f"{n}.npz")["gt"]
        src = rgb_of(fid)
        row = Image.new("RGB", (960, 252), R.SURFACE)
        iou = comma10k_ious(tt(fid, "bgr"), gt)          # the API's class map is already at the crop's size
        a = tile(fid, "bgr", 480, 252, f"comma10k {n}: p150 (BGR) | fg IoU vs GT {iou['foreground']:.2f}", fs=12)
        b = Image.fromarray(R.overlay(R.resize_rgb(src, 480, 252), R.resize_cls(gt, 480, 252)))
        R.chip_label(b, (6, 6), "ground truth (comma10k labels mapped to 3 classes)", fs=12)
        row.paste(a, (0, 0))
        row.paste(b, (480, 0))
        ImageDraw.Draw(row).line([(480, 0), (480, 252)], fill=R.SURFACE, width=2)
        parts.append(row)
    bar = R.caption_bar(960, "SceneSeg on comma10k (shipped samples; SceneSeg training data) - p150 output vs ground truth",
                        [f"{TT}; deployed BGR order; training ROI crop. comma10k is one of SceneSeg's training sets "
                         "(5 of the 6 shipped images are training images), so this is a pipeline check, not a "
                         "held-out score."], footer=C10K_FOOT)
    save(R.stack(parts + [bar]), "sceneseg_comma10k_tt_vs_gt.jpg")

    # --- PandaSet
    fid = "ps019_f40_front_camera"
    img = tile(fid, "bgr", 960, 540)
    bar = R.caption_bar(960, "SceneSeg on PandaSet - p150 output",
                        [f"PandaSet seq 019 ({R.PS_WHERE['019']}), frame 40, front_camera 1920x1080, 52 deg HFOV "
                         f"(the VisionPilot spec is 52-55 deg). {TT}; deployed BGR order, squashed to 640x320; "
                         f"foreground {R.fgpct(tt(fid, 'bgr')):.1f} % of pixels; {ag(fid, 'bgr')}."], footer=PS_FOOT)
    save(R.stack([img, bar]), "sceneseg_pandaset_019_f40_front_tt.jpg")
    tt_vs_cpu(fid, "bgr", "sceneseg_pandaset_019_f40_tt_vs_cpu.jpg",
              "p150 vs fp32 CPU reference on PandaSet 019, frame 40, front camera", PS_FOOT, w=480, h=270)

    grid = Image.new("RGB", (960, 540), R.SURFACE)
    for i, s in enumerate(("019", "005", "029", "090")):
        f = f"ps{s}_f40_front_camera"
        grid.paste(tile(f, "bgr", 480, 270, f"seq {s}: {R.PS_WHERE[s]}  |  fg {R.fgpct(tt(f, 'bgr')):.1f} %"),
                   ((i % 2) * 480, (i // 2) * 270))
    d = ImageDraw.Draw(grid)
    d.line([(480, 0), (480, 540)], fill=R.SURFACE, width=2)
    d.line([(0, 270), (960, 270)], fill=R.SURFACE, width=2)
    worst = min(rows[f"ps{s}_f40_front_camera_bgr"]["mask_agreement"] for s in ("019", "005", "029", "090"))
    bar = R.caption_bar(960, "SceneSeg on four PandaSet sequences (frame 40, front_camera) - p150 output",
                        [f"{TT}; deployed BGR order; fg = share of pixels in the published foreground mask. "
                         f"Published mask agreement with the fp32 CPU reference >= {100 * worst:.2f} %."],
                        footer=PS_FOOT)
    save(R.stack([grid, bar]), "sceneseg_pandaset_4seq_f40_front_tt.jpg")

    parts = []
    for s in ("019", "056", "065"):
        f = f"ps{s}_f40_front_camera"
        row = Image.new("RGB", (960, 270), R.SURFACE)
        for j, v in enumerate(VARIANTS):
            name = "bgr: deployed order (default)" if v == "bgr" else "rgb: training order (option)"
            row.paste(tile(f, v, 480, 270, f"{name}  |  seq {s}  |  fg {R.fgpct(tt(f, v)):.1f} %"), (j * 480, 0))
        ImageDraw.Draw(row).line([(480, 0), (480, 270)], fill=R.SURFACE, width=2)
        parts.append(row)
    bar = R.caption_bar(960, "Channel order: deployed BGR vs training-order RGB - p150 output, both serve profiles", [
        "VisionPilot's C++ pre-processing feeds B, G, R planes to a network trained on R, G, B. Rows: PandaSet 019 "
        "(SF Embarcadero), 056 (SF bike lanes: cyclists are mostly missed in BGR) and 065 (night), frame 40, "
        f"front_camera. {TT}."], footer=PS_FOOT)
    save(R.stack(parts + [bar]), "sceneseg_pandaset_bgr_vs_rgb_tt.jpg")

    # --- PandaSet 019 clip (GIF, 640 px, every 2nd frame = 5 fps over the same 8 s)
    clip = []
    for f in range(0, 80, 2):
        fid = f"ps019_f{f:02d}_front_camera"
        img = tile(fid, "bgr", 640, 360, f"frame {f:02d}/79   t = {f / 10:4.1f} s   fg {R.fgpct(tt(fid, 'bgr')):.1f} %",
                   fs=11)
        clip.append(img)
    worst = min(rows[f"ps019_f{f:02d}_front_camera_bgr"]["mask_agreement"] for f in range(80))
    bar = R.caption_bar(640, "SceneSeg on PandaSet seq 019, 8 s at 10 Hz (every 2nd frame shown) - p150 output",
                        [f"{TT}; deployed BGR order; every frame independent. Published mask agreement with the fp32 "
                         f"CPU reference >= {100 * worst:.2f} % on all 80 frames."], footer=PS_FOOT, fs=12)
    frames_ = [R.stack([c, bar]).convert("P", palette=Image.ADAPTIVE, colors=255, dither=Image.Dither.NONE)
               for c in clip]
    p = media / "sceneseg_pandaset_019_front_tt.gif"
    frames_[0].save(p, save_all=True, append_images=frames_[1:], duration=200, loop=0, optimize=True, disposal=1)
    print("wrote", p, frames_[0].size, len(frames_), "frames", p.stat().st_size, flush=True)

    # --- nuScenes (non-commercial)
    fid = "ns0103_k28_CAM_FRONT"
    img = tile(fid, "bgr", 960, 540)
    bar = R.caption_bar(960, "SceneSeg on nuScenes (NON-COMMERCIAL, CC BY-NC-SA 4.0) - p150 output",
                        ["nuScenes v1.0-mini scene-0103 (Boston Seaport, mini_val), key-frame 28, CAM_FRONT 1600x900, "
                         f"65 deg HFOV (wider than the 52-55 deg VisionPilot spec). {TT}; deployed BGR order; "
                         f"foreground {R.fgpct(tt(fid, 'bgr')):.1f} % of pixels; {ag(fid, 'bgr')}."], footer=NS_FOOT)
    save(R.stack([img, bar]), "sceneseg_nuscenes_scene0103_k28_front_tt_NC.jpg")

    # --- KITTI (non-commercial): p150 vs ground truth
    from kitti_semantics import load_semantic, to_sceneseg       # research loader (datasets/tools)

    picks = json.loads(KITTI_PICKS.read_text())["kitti_frames"]
    parts, ious = [], []
    for k in picks:
        fid = f"kitti_{k}_c2to1"
        src = rgb_of(fid)
        gt = _crop_view(to_sceneseg(load_semantic(str(R.C.KITTI_ROOT), k)), "c2to1")
        pred = tt(fid, "bgr")
        u = np.logical_or(pred == 1, gt == 1).sum()
        iou = float(np.logical_and(pred == 1, gt == 1).sum() / u) if u else 1.0
        ious.append(iou)
        row = Image.new("RGB", (960, 240), R.SURFACE)
        a = tile(fid, "bgr", 480, 240, f"{k}  p150 prediction (BGR)  |  fg IoU {iou:.2f}", fs=12)
        b = Image.fromarray(R.overlay(R.resize_rgb(src, 480, 240), R.resize_cls(gt, 480, 240)))
        R.chip_label(b, (6, 6), "ground truth (KITTI labels mapped to 3 classes)", fs=12)
        row.paste(a, (0, 0))
        row.paste(b, (480, 0))
        ImageDraw.Draw(row).line([(480, 0), (480, 240)], fill=R.SURFACE, width=2)
        parts.append(row)
    bar = R.caption_bar(960, "SceneSeg on KITTI (held-out, NON-COMMERCIAL CC BY-NC-SA 3.0) - p150 output vs ground truth",
                        ["Rows: the frames at the 85th, 50th and 15th percentile of the CPU reference's per-frame "
                         f"foreground IoU (deployed BGR, 2:1 centre crop). {TT}."], footer=KITTI_FOOT)
    save(R.stack(parts + [bar]), "sceneseg_kitti_heldout_tt_NC.jpg")
    (out / "render.json").write_text(json.dumps({"kitti_frames": picks, "kitti_fg_iou_tt": ious}, indent=1) + "\n")


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("step", choices=("run", "render", "cls"))
    ap.add_argument("--out", type=Path, required=True)
    ap.add_argument("--media", type=Path, default=CODE.parent / "media")
    a = ap.parse_args()
    if a.step == "run":
        run(a.out)
    elif a.step == "cls":
        cls_maps(a.out)
    else:
        render(a.out, a.media)


if __name__ == "__main__":
    main()