# SPDX-License-Identifier: Apache-2.0 """Host pre-processing of METEOR's runtimes: eight camera frames + calibration + ego speed -> the graph's feed. The contract (research/meteor/SPEC.md sections 2-3; ``meteor_v157.param.yaml``; tier4/METEOR ``hf/onnx_smoke_test.py: 34-56``, the stated reference for feeding the graph, and ``deploy/cpp/realtime_main.cpp:123-140``): - ``imgs`` uint8 [1, 8, 3, 432, 768], RGB, CHW, cameras in the training order (``config.CAMERAS``). A frame of another size is resized with OpenCV ``INTER_AREA`` (``ttaw.image_area.meteor_resize``, bit-exact), anisotropically if the aspect differs; frames are raw and unrectified (training never undistorted). - ``K`` float32 [1, 8, 3, 3] at 768x432: the intrinsics of the image as captured, row 0 x 768 / w0 and row 1 x 432 / h0 (``extract_gt.py:128-130``); only fx, fy, cx, cy reach the graph. - ``T_cam_ego`` float32 [1, 8, 4, 4] = inv(T_ego_cam): ``np.linalg.inv`` on float32 like ``onnx_smoke_test.py:46`` (the research goldens' feed, byte-identical; ``inverse="float64"`` gives the C++ runtime's double inverse cast to float). - ``v0`` float32 [1]: the ego speed in m/s. - An absent camera (the 7-camera rig, or a dropped slot) is an all-zero image plus the DONOR's K and pose (``bevlane/dataset.py:17-21``: CAM_BACK_NARROW <- CAM_BACK_WIDE, CAM_FRONT_NARROW <- CAM_FRONT_WIDE). With the released ``/255`` input a uint8 zero image is exactly the trained "absent" input; with ``input_norm="imagenet"`` the camera must be zeroed after normalising (``present`` mask, :func:`normalize_images`). :func:`load_scene_frame` reads METEOR's demo-scene layout (``manifest.json``, ``img/NNNN_.jpg``, ``ego_motion.npz``), which the public-dataset samples reuse; it reproduces ``research/meteor/scripts/meteor_io.py``'s feed byte for byte (``tests/test_host_preprocess.py`` checks the sha256 of the shipped sample against its golden). :func:`assemble_frame` is the API path (images of any size + calibration + speed). """ from __future__ import annotations import hashlib import json import math import os from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, Mapping, Optional, Sequence, Tuple import numpy as np from ..reference import config as C from ..ttaw.image_area import meteor_resize, scale_intrinsics from ..ttaw.io import load_image __all__ = ["MeteorFrame", "assemble_frame", "load_scene_frame", "load_manifest", "scene_dir_of", "invert_extrinsics", "prepare_camera", "normalize_images", "pose_from_transform"] F32 = np.float32 @dataclass class MeteorFrame: """One network input (the ONNX feed) plus what the host post-processing needs.""" imgs: np.ndarray # uint8 [1, 8, 3, 432, 768] RGB K: np.ndarray # float32 [1, 8, 3, 3] at 768x432 T_cam_ego: np.ndarray # float32 [1, 8, 4, 4] ego -> camera v0: np.ndarray # float32 [1] m/s present: np.ndarray # bool [8] pose: Optional[Tuple[float, float, float]] = None # ego (x, y, yaw) in a world frame (seg fusion, yaw tracks) meta: Dict[str, Any] = field(default_factory=dict) def feed(self) -> Dict[str, np.ndarray]: """The ONNX input dict (``imgs``, ``K``, ``T_cam_ego``, ``v0``).""" return {"imgs": self.imgs, "K": self.K, "T_cam_ego": self.T_cam_ego, "v0": self.v0} def sha256(self) -> Dict[str, str]: return {k: hashlib.sha256(np.ascontiguousarray(v).tobytes()).hexdigest() for k, v in self.feed().items()} def invert_extrinsics(T_ego_cam: Any, inverse: str = "float32") -> np.ndarray: """camera -> ego (4x4) to the graph's ego -> camera, float32. ``float32``: ``np.linalg.inv`` of the float32 matrix (``onnx_smoke_test.py:46``, the research feed); ``float64``: double inverse cast to float (``realtime_main.cpp: 123-128``).""" if inverse == "float32": return np.linalg.inv(np.asarray(T_ego_cam, F32).reshape(4, 4)).astype(F32) if inverse == "float64": return np.linalg.inv(np.asarray(T_ego_cam, np.float64).reshape(4, 4)).astype(F32) raise ValueError(f"inverse must be 'float32' or 'float64', not {inverse!r}") def prepare_camera(image: Any, K: Optional[Any] = None) -> Tuple[np.ndarray, Optional[np.ndarray], Tuple[int, int]]: """One frame (anything ``ttaw.io.load_image`` takes; RGB) -> (uint8 [3, 432, 768] RGB CHW, K at 768x432 or None, the source (h, w)). INTER_AREA when the size differs (``ttaw.image_area``); K is scaled per axis.""" rgb = load_image(image) src_hw = (int(rgb.shape[0]), int(rgb.shape[1])) if src_hw != (C.IMG_H, C.IMG_W): if src_hw[0] < C.IMG_H or src_hw[1] < C.IMG_W: raise ValueError(f"camera frame {src_hw[1]}x{src_hw[0]} is smaller than METEOR's 768x432 input " "(INTER_AREA up-scaling is not supported)") rgb = meteor_resize(rgb) k = None if K is None else scale_intrinsics(K, src_hw) return np.ascontiguousarray(rgb.transpose(2, 0, 1)), k, src_hw def pose_from_transform(T_world_from_ego: Any) -> Tuple[float, float, float]: """(x, y, yaw) of a 4x4 world <- ego transform (METEOR's ``ego_motion.npz`` ``pose`` convention).""" T = np.asarray(T_world_from_ego, np.float64).reshape(4, 4) return float(T[0, 3]), float(T[1, 3]), float(math.atan2(T[1, 0], T[0, 0])) def assemble_frame(images: Mapping[str, Any], intrinsics: Mapping[str, Any], T_ego_cam: Mapping[str, Any], v0: float, *, present: Optional[Mapping[str, bool]] = None, inverse: str = "float32", pose: Optional[Tuple[float, float, float]] = None) -> MeteorFrame: """The API path: per camera name an image (any size; None = absent), the intrinsics of that image and its camera -> ego transform; ``v0`` in m/s. Absent cameras get a zero image and their donor's K and pose; a camera without a donor (only the two narrow cameras have one) must be present. A narrow camera sent as an all-zero image is absent too (METEOR's data convention, ``absent_zero.jpg``): its calibration is replaced by the donor's, as in training (``bevlane/dataset.py:131-145``).""" imgs = np.zeros((1, C.N_CAMS, 3, C.IMG_H, C.IMG_W), np.uint8) K = np.zeros((1, C.N_CAMS, 3, 3), F32) T = np.zeros((1, C.N_CAMS, 4, 4), F32) flags = np.zeros(C.N_CAMS, bool) src_sizes: Dict[str, Any] = {} cams: Dict[str, Tuple[np.ndarray, np.ndarray]] = {} for name in C.CAMERAS: img = images.get(name) ok = img is not None and (present is None or bool(present.get(name, True))) if not ok: continue chw, k, src_hw = prepare_camera(img, intrinsics.get(name)) if name in C.DONOR and not chw.any(): continue # an all-zero narrow camera is METEOR's "absent" convention: zero + donor pose if k is None or T_ego_cam.get(name) is None: raise ValueError(f"{name}: intrinsics and T_ego_cam are required for a present camera") i = C.CAMERAS.index(name) imgs[0, i] = chw flags[i] = True cams[name] = (k, np.asarray(T_ego_cam[name], np.float64).reshape(4, 4)) src_sizes[name] = list(src_hw) donors: Dict[str, str] = {} for i, name in enumerate(C.CAMERAS): src = name if flags[i] else C.DONOR.get(name) if src is None or src not in cams: raise ValueError(f"{name} is absent and has no present donor camera (donors: {C.DONOR})") if src != name: donors[name] = src k, t = cams[src] K[0, i] = np.asarray(k, F32) T[0, i] = invert_extrinsics(t, inverse) return MeteorFrame(imgs, K, T, np.array([v0], F32), flags, pose, {"source_hw": src_sizes, "donors": donors, "inverse": inverse}) def scene_dir_of(root: os.PathLike) -> Path: """``/scenes.txt`` -> ``/`` (METEOR demo-root layout).""" root = Path(root) return root / (root / "scenes.txt").read_text().split()[0] def load_manifest(scene_dir: os.PathLike) -> Dict[str, Any]: return json.loads((Path(scene_dir) / "manifest.json").read_text()) def load_scene_frame(scene_dir: os.PathLike, frame: int, manifest: Optional[Mapping[str, Any]] = None, *, inverse: str = "float32") -> MeteorFrame: """Frame ``frame`` of a METEOR demo-scene directory (``manifest.json`` cams[c].K at 768x432 and T_ego_cam, frames[i].imgs[c]; ``ego_motion.npz`` v0 / pose / wp / valid), as ``meteor_io.load_frame`` feeds the graph.""" scene_dir = Path(scene_dir) m = manifest or load_manifest(scene_dir) fr = m["frames"][frame] imgs = np.zeros((1, C.N_CAMS, 3, C.IMG_H, C.IMG_W), np.uint8) K = np.zeros((1, C.N_CAMS, 3, 3), F32) T = np.zeros((1, C.N_CAMS, 4, 4), F32) for i, cam in enumerate(C.CAMERAS): chw, _, _ = prepare_camera(scene_dir / fr["imgs"][cam]) imgs[0, i] = chw K[0, i] = np.asarray(m["cams"][cam]["K"], F32) T[0, i] = invert_extrinsics(m["cams"][cam]["T_ego_cam"], inverse) present = np.asarray(m.get("present_mask", [1] * C.N_CAMS), bool).reshape(C.N_CAMS) ego = np.load(scene_dir / "ego_motion.npz") v0 = float(ego["v0"][frame]) pose = tuple(float(v) for v in ego["pose"][frame]) if "pose" in ego.files else None meta: Dict[str, Any] = {"scene": m.get("scene", scene_dir.name), "frame": int(frame)} for key in ("wp", "valid"): if key in ego.files: meta[key] = np.asarray(ego[key][frame]).tolist() return MeteorFrame(imgs, K, T, np.array([v0], F32), present, pose, meta) def normalize_images(imgs_u8: np.ndarray, mode: str = "onnx", present: Optional[Sequence[bool]] = None) -> np.ndarray: """Host oracle of the device input normalisation: uint8 [1, 8, 3, H, W] -> float32 [8, 3, H, W]; ``onnx`` = x / 255 (as released), ``imagenet`` = (x / 255 - mean) / std with absent cameras zeroed afterwards (D12).""" x = np.asarray(imgs_u8).astype(F32) / F32(255.0) if mode == "imagenet": mean = np.asarray(C.IMAGENET_MEAN, F32).reshape(1, 1, 3, 1, 1) std = np.asarray(C.IMAGENET_STD, F32).reshape(1, 1, 3, 1, 1) x = (x - mean) / std if present is not None: x = x * np.asarray(present, F32).reshape(1, -1, 1, 1, 1) elif mode != "onnx": raise ValueError(f"input_norm must be one of {C.INPUT_NORMS}, not {mode!r}") return x.reshape(C.N_CAMS, 3, C.IMG_H, C.IMG_W)