/** * Interactive top-down BEV canvas for one perception frame: GT boxes, camera * detections with along-ray uncertainty ellipses, LiDAR detections, fused * boxes (masklet-propagated ones dashed), with hover details and * click-to-follow-track across frames. Built on the shared CanvasSurface * (pan/zoom) primitive. */ import { useMemo, useState } from 'react'; import Box from '@mui/material/Box'; import Chip from '@mui/material/Chip'; import Typography from '@mui/material/Typography'; import CanvasSurface from '../visual/canvas'; import type { BevDetection, BevFusedBox, BevGtBox, BevReplayFrame } from '../../services/bevfusion'; import { tokens } from '../../theme'; export const LAYER_META = { gt: { label: 'Ground truth', color: '#e6e9ec' }, camera: { label: 'Camera dets + uncertainty', color: '#ffa726' }, lidar: { label: 'LiDAR dets', color: '#26c6da' }, fused: { label: 'Fused (tracked)', color: '#66bb6a' }, masklet: { label: 'Masklet-propagated', color: '#ab47bc' }, baseline: { label: 'Camera baseline', color: '#ef5350' }, } as const; export type LayerId = keyof typeof LAYER_META; const CLASS_COLORS: Record = { vehicle: '#66bb6a', truck: '#9ccc65', pedestrian: '#ff7043', cyclist: '#ffca28', motorcycle: '#ec407a', }; export interface HoverInfo { kind: 'gt' | 'camera' | 'lidar' | 'fused' | 'baseline'; title: string; lines: string[]; } // World: x forward 0..80m (drawn rightward), y left/right ±25m (drawn vertically). const WORLD = { x: -4, y: -25, w: 88, h: 50 }; /** 1-sigma covariance ellipse parameters from a 2x2 covariance matrix. */ function covEllipse(cov: [[number, number], [number, number]]): { rx: number; ry: number; angleDeg: number } { const [[a, b], [, d]] = cov; const tr = a + d; const det = a * d - b * b; const disc = Math.sqrt(Math.max(0, (tr * tr) / 4 - det)); const l1 = tr / 2 + disc; const l2 = tr / 2 - disc; const angle = Math.atan2(l1 - a, b || 1e-9); return { rx: Math.sqrt(Math.max(l1, 1e-6)), ry: Math.sqrt(Math.max(l2, 1e-6)), angleDeg: (angle * 180) / Math.PI }; } function OrientedBox({ x, y, l, w, yaw, stroke, fill = 'none', dashed = false, strokeWidth = 0.22, onEnter, onLeave, onClick, highlight = false, }: { x: number; y: number; l: number; w: number; yaw: number; stroke: string; fill?: string; dashed?: boolean; strokeWidth?: number; onEnter?: () => void; onLeave?: () => void; onClick?: () => void; highlight?: boolean; }) { // ego frame: +x forward, +y left. Screen: x → right, y → down = -world y. const deg = (-yaw * 180) / Math.PI; return ( { if (onClick) { e.stopPropagation(); onClick(); } }} style={{ cursor: onClick ? 'pointer' : undefined }} > {/* heading tick */} ); } interface BevFrameCanvasProps { frame: BevReplayFrame; layers: Record; selectedTrackId: number | string | null; onSelectTrack: (id: number | string | null) => void; /** Track trail of the selected track: positions up to the current frame. */ trail: Array<{ x: number; y: number }>; onHover: (info: HoverInfo | null) => void; height?: number; } export default function BevFrameCanvas({ frame, layers, selectedTrackId, onSelectTrack, trail, onHover, height = 460 }: BevFrameCanvasProps) { const [hoverKey, setHoverKey] = useState(null); const hover = (key: string | null, info: HoverInfo | null) => { setHoverKey(key); onHover(info); }; const gtBoxes = useMemo(() => frame.gt, [frame]); return ( onSelectTrack(null)}> {() => ( <> {/* range rings + ego */} {[20, 40, 60, 80].map((r) => ( {r}m ))} {/* ego vehicle */} EGO {/* selected-track trail */} {trail.length > 1 ? ( `${p.x},${-p.y}`).join(' ')} fill="none" stroke="#fff" strokeWidth={0.18} strokeDasharray="0.5 0.5" opacity={0.8} /> ) : null} {/* GT */} {layers.gt ? gtBoxes.map((g: BevGtBox) => { const [x, y, , l, w, , yaw] = g.bbox_3d; const key = `gt-${g.instance_id}`; return ( hover(key, { kind: 'gt', title: `GT · ${g.class_name}`, lines: [ `instance ${g.instance_id.split('-').slice(-2).join('-')}`, `position (${x.toFixed(1)}, ${y.toFixed(1)}) m · ${g.distance.toFixed(1)} m away`, g.occluded ? 'camera line-of-sight OCCLUDED this frame' : 'visible to camera', ], }) } onLeave={() => hover(null, null)} /> ); }) : null} {/* camera detections + uncertainty ellipses */} {layers.camera ? frame.camera.map((d: BevDetection, i: number) => { const e = covEllipse(d.cov); const key = `cam-${i}`; return ( hover(key, { kind: 'camera', title: `Camera det · ${d.class_name}`, lines: [ `position (${d.x.toFixed(1)}, ${d.y.toFixed(1)}) m · conf ${d.confidence.toFixed(2)}`, `1σ uncertainty ${e.rx.toFixed(1)} m along ray × ${e.ry.toFixed(1)} m across`, 'monocular depth: uncertainty stretches along the camera ray', ], }) } onPointerLeave={() => hover(null, null)} > ); }) : null} {/* lidar detections */} {layers.lidar ? frame.lidar.map((d: BevDetection, i: number) => { const key = `lid-${i}`; const s = 0.55; return ( hover(key, { kind: 'lidar', title: `LiDAR det · ${d.class_name}`, lines: [ `position (${d.x.toFixed(1)}, ${d.y.toFixed(1)}) m · conf ${d.confidence.toFixed(2)}`, 'tight isotropic position uncertainty; weak semantics (geometry templates)', ], }) } onPointerLeave={() => hover(null, null)} > ); }) : null} {/* camera baseline boxes */} {layers.baseline ? frame.baseline.map((b: BevFusedBox, i: number) => { const [x, y, , l, w, , yaw] = b.bbox_3d; const key = `base-${i}`; return ( hover(key, { kind: 'baseline', title: `Baseline (camera-only) · ${b.class_name}`, lines: [ `track #${b.track_id} · conf ${b.confidence.toFixed(2)}`, 'greedy frame-to-frame association — IDs churn on dropouts', ], }) } onLeave={() => hover(null, null)} /> ); }) : null} {/* fused boxes (masklet propagated = dashed) */} {frame.fused.map((b: BevFusedBox, i: number) => { const show = b.propagated ? layers.masklet : layers.fused; if (!show) return null; const [x, y, , l, w, , yaw] = b.bbox_3d; const key = `fused-${i}`; const selected = selectedTrackId !== null && b.track_id === selectedTrackId; const color = b.propagated ? LAYER_META.masklet.color : CLASS_COLORS[b.class_name] ?? LAYER_META.fused.color; return ( hover(key, { kind: 'fused', title: `Fused · ${b.class_name}${b.propagated ? ' (masklet-propagated)' : ''}`, lines: [ `track #${b.track_id} · conf ${b.confidence.toFixed(2)} · (${x.toFixed(1)}, ${y.toFixed(1)}) m`, b.propagated ? 'no detection this frame — box carried by the masklet motion model' : 'decoded from the fused camera+LiDAR BEV grid', 'click to follow this track across frames', ], }) } onLeave={() => hover(null, null)} onClick={() => onSelectTrack(selected ? null : b.track_id)} /> #{b.track_id} ); })} )} ); } export function CanvasLegend({ layers, onToggle }: { layers: Record; onToggle: (l: LayerId) => void }) { return ( {(Object.keys(LAYER_META) as LayerId[]).map((l) => ( onToggle(l)} sx={{ height: 22, fontSize: 11, cursor: 'pointer', bgcolor: layers[l] ? `${LAYER_META[l].color}26` : 'transparent', color: layers[l] ? LAYER_META[l].color : tokens.color.textFaint, border: `1px solid ${layers[l] ? LAYER_META[l].color : tokens.color.border}`, textDecoration: layers[l] ? 'none' : 'line-through', transition: `all ${tokens.motion.fast}`, }} /> ))} scroll to zoom · drag background to pan · hover any object · click a fused box to follow its track ); }