sensorflow/vitis/render.py

"""Render bevfusion synthetic scenes to schematic sensor images.

The bevfusion package (imported read-only) provides deterministic scenes
with 3D ground truth, planted occlusion windows, and day/night/rain
conditions. This module rasterizes those scenes into two image spaces:

* BEV top-down grayscale images — one pixel maps linearly to ego-frame
  meters, so dense optical flow in BEV pixel space is directly a motion
  field in the ego frame (used by HIL detection and temporal profiling).
* Front-camera stereo pairs — pinhole projection with a horizontal baseline,
  so block-matching disparity converts back to metric depth (used by the
  stereo consistency check).

These are schematic renderings (intensity-coded boxes plus sensor noise),
not photorealistic frames; they exist so the acceleration backends process
images whose ground truth is exactly known.
"""

from __future__ import annotations

from typing import Dict, List, Tuple

import numpy as np

from sensorflow.bevfusion.scenes import SceneFrame, SceneSequence

# BEV raster geometry: ego-frame x in [0, 80] m forward, y in [-25, 25] m.
BEV_H, BEV_W = 192, 256
X_MIN, X_MAX = 0.0, 80.0
Y_MIN, Y_MAX = -25.0, 25.0

CLASS_INTENSITY = {"vehicle": 0.85, "truck": 0.95, "pedestrian": 0.55,
                   "cyclist": 0.65, "motorcycle": 0.75}

# Front stereo camera model.
FOCAL_PX = 200.0
BASELINE_M = 0.5
CAM_H, CAM_W = 128, 256
CAM_HEIGHT_M = 1.6


def ego_to_bev_px(x: float, y: float) -> Tuple[float, float]:
    """Ego meters -> (row, col) in the BEV raster. Row 0 is the far field."""
    row = (X_MAX - x) / (X_MAX - X_MIN) * (BEV_H - 1)
    col = (y - Y_MIN) / (Y_MAX - Y_MIN) * (BEV_W - 1)
    return row, col


def bev_px_to_ego(row: float, col: float) -> Tuple[float, float]:
    x = X_MAX - row / (BEV_H - 1) * (X_MAX - X_MIN)
    y = Y_MIN + col / (BEV_W - 1) * (Y_MAX - Y_MIN)
    return x, y


BEV_M_PER_PX_X = (X_MAX - X_MIN) / (BEV_H - 1)   # meters per row
BEV_M_PER_PX_Y = (Y_MAX - Y_MIN) / (BEV_W - 1)   # meters per col


def _condition_params(seq: SceneSequence) -> Dict:
    scale = 1.0
    noise = 0.012
    if seq.time_of_day == "night":
        scale = 0.45     # dimmer returns -> fewer effective signal bits
        noise = 0.02
    if seq.weather == "rain":
        scale *= 0.8
        noise += 0.015
    return {"intensity_scale": scale, "noise_sigma": noise}


def render_bev_frame(frame: SceneFrame, seq: SceneSequence, seed: int,
                     include_occluded: bool = True) -> np.ndarray:
    """Grayscale BEV raster (BEV_H, BEV_W) in [0, 1] with per-frame noise.

    Noise is seeded from (seed, frame index) so BOTH backends see the exact
    same source pixels — every ref-vs-vitis difference downstream is caused
    by the backend, never by the renderer.
    """
    params = _condition_params(seq)
    img = np.zeros((BEV_H, BEV_W), dtype=np.float32)
    for gt in frame.gt:
        if gt.occluded and not include_occluded:
            continue
        x, y, _, l, w, _, _ = gt.bbox_3d
        r, c = ego_to_bev_px(x, y)
        hl = max(1.5, l / 2.0 / BEV_M_PER_PX_X)
        hw = max(1.5, w / 2.0 / BEV_M_PER_PX_Y)
        r0, r1 = int(max(0, r - hl)), int(min(BEV_H, r + hl + 1))
        c0, c1 = int(max(0, c - hw)), int(min(BEV_W, c + hw + 1))
        inten = CLASS_INTENSITY.get(gt.class_name, 0.7) * params["intensity_scale"]
        if gt.occluded:
            inten *= 0.55  # partially attenuated return
        img[r0:r1, c0:c1] = np.maximum(img[r0:r1, c0:c1], inten)
    rng = np.random.default_rng((seed * 1_000_003 + frame.index * 7919) % (2**63))
    img += rng.normal(0.0, params["noise_sigma"], img.shape).astype(np.float32)
    img += 0.04  # ground-return floor
    return np.clip(img, 0.0, 1.0)


def render_bev_rgb(frame: SceneFrame, seq: SceneSequence, seed: int) -> np.ndarray:
    """Color BEV raster (class-hued boxes) used as ISP/augmentation source."""
    gray = render_bev_frame(frame, seq, seed)
    hue = {"vehicle": (1.0, 0.85, 0.6), "truck": (1.0, 0.7, 0.4),
           "pedestrian": (0.6, 1.0, 0.7), "cyclist": (0.6, 0.85, 1.0),
           "motorcycle": (0.9, 0.7, 1.0)}
    rgb = np.stack([gray, gray, gray], axis=-1)
    for gt in frame.gt:
        x, y, _, l, w, _, _ = gt.bbox_3d
        r, c = ego_to_bev_px(x, y)
        hl = max(1.5, l / 2.0 / BEV_M_PER_PX_X)
        hw = max(1.5, w / 2.0 / BEV_M_PER_PX_Y)
        r0, r1 = int(max(0, r - hl)), int(min(BEV_H, r + hl + 1))
        c0, c1 = int(max(0, c - hw)), int(min(BEV_W, c + hw + 1))
        tint = np.array(hue.get(gt.class_name, (1.0, 1.0, 1.0)), dtype=np.float32)
        rgb[r0:r1, c0:c1] *= tint
    return np.clip(rgb, 0.0, 1.0)


def render_stereo_pair(frame: SceneFrame, seq: SceneSequence,
                       seed: int) -> Tuple[np.ndarray, np.ndarray, List[Dict]]:
    """Front-camera stereo pair + per-object projection records.

    Left/right pinhole cameras separated by BASELINE_M; the right image sees
    each object shifted left by disparity = FOCAL_PX * BASELINE_M / depth.
    Returns (left, right, objects) where objects carry
    {instance_id, class_name, depth_m, u, v, disparity_px}.
    """
    params = _condition_params(seq)
    cx, cy = CAM_W / 2.0, CAM_H / 2.0
    left = np.zeros((CAM_H, CAM_W), dtype=np.float32)
    right = np.zeros((CAM_H, CAM_W), dtype=np.float32)
    objects: List[Dict] = []
    # Draw far to near so nearer objects overwrite (correct occlusion order).
    for gt in sorted(frame.gt, key=lambda g: -g.bbox_3d[0]):
        if gt.occluded:
            continue
        x, y, z, l, w, h, _ = gt.bbox_3d
        if x < 3.0:
            continue
        depth = float(x)
        u = FOCAL_PX * (-y) / depth + cx
        v = FOCAL_PX * (CAM_HEIGHT_M - z) / depth + cy
        half_w = max(1.5, FOCAL_PX * (w / 2.0) / depth)
        half_h = max(1.5, FOCAL_PX * (h / 2.0) / depth)
        disparity = FOCAL_PX * BASELINE_M / depth
        inten = CLASS_INTENSITY.get(gt.class_name, 0.7) * params["intensity_scale"]
        # Textured fill so block matching has gradients to lock onto.
        rng_tex = np.random.default_rng(abs(hash(gt.instance_id)) % (2**63))
        for img, du in ((left, 0.0), (right, -disparity)):
            u0 = int(max(0, u + du - half_w))
            u1 = int(min(CAM_W, u + du + half_w + 1))
            v0 = int(max(0, v - half_h))
            v1 = int(min(CAM_H, v + half_h + 1))
            if u1 <= u0 or v1 <= v0:
                continue
            tex = rng_tex.uniform(0.7, 1.0, (v1 - v0, u1 - u0)).astype(np.float32)
            # Same texture crop in both views (rng reset below).
            img[v0:v1, u0:u1] = inten * tex
            rng_tex = np.random.default_rng(abs(hash(gt.instance_id)) % (2**63))
        if 0 <= u < CAM_W and 0 <= v < CAM_H:
            objects.append({"instance_id": gt.instance_id,
                            "class_name": gt.class_name,
                            "depth_m": round(depth, 3),
                            "u": round(u, 2), "v": round(v, 2),
                            "disparity_px": round(disparity, 3)})
    rng = np.random.default_rng((seed * 999_983 + frame.index * 104_729) % (2**63))
    for img in (left, right):
        img += rng.normal(0.0, params["noise_sigma"], img.shape).astype(np.float32)
        img += 0.03
    return (np.clip(left, 0.0, 1.0), np.clip(right, 0.0, 1.0), objects)