| export type PlaneId = |
| | "ops" |
| | "physical" |
| | "perception" |
| | "prediction" |
| | "planning" |
| | "safety" |
| | "learning"; |
|
|
| export type StageId = |
| | "mission" |
| | "sensors" |
| | "fusion" |
| | "perception" |
| | "prediction" |
| | "behavior" |
| | "motion" |
| | "control" |
| | "actuators" |
| | "world" |
| | "supervisor" |
| | "health" |
| | "fallback" |
| | "capture" |
| | "mining" |
| | "labeling" |
| | "training" |
| | "evaluation" |
| | "simulation" |
| | "release"; |
|
|
| export type Stage = { |
| id: StageId; |
| title: string; |
| plane: PlaneId; |
| role: string; |
| detail: string; |
| inputs: string[]; |
| outputs: string[]; |
| mlRole: string; |
| failureModes: string[]; |
| keyIdea: string; |
| }; |
|
|
| export const PLANES: Record<PlaneId, { label: string; accent: string; blurb: string }> = { |
| ops: { |
| label: "Human / Ops", |
| accent: "#f0abfc", |
| blurb: "Mission intent and monitoring. The stack serves a goal someone set.", |
| }, |
| physical: { |
| label: "Physical", |
| accent: "#7dd3fc", |
| blurb: "Sensors, compute, actuators, vehicle. Everything with a clock and a wire.", |
| }, |
| perception: { |
| label: "Perception", |
| accent: "#5eead4", |
| blurb: "Turn sensor data into a representation of the world.", |
| }, |
| prediction: { |
| label: "Prediction", |
| accent: "#fbbf24", |
| blurb: "Estimate what the world will do next — as a distribution, not a guess.", |
| }, |
| planning: { |
| label: "Planning & Control", |
| accent: "#c4b5fd", |
| blurb: "Decide what the vehicle should do, then actually execute it.", |
| }, |
| safety: { |
| label: "Safety & Supervision", |
| accent: "#fb7185", |
| blurb: "Detect failures, constrain behavior, trigger fallbacks. Deterministic by design.", |
| }, |
| learning: { |
| label: "Data & Learning", |
| accent: "#34d399", |
| blurb: "Collect failures, label, train, evaluate, deploy. This is what makes the stack improve.", |
| }, |
| }; |
|
|
| export const RUNTIME_ORDER: StageId[] = [ |
| "sensors", |
| "fusion", |
| "perception", |
| "prediction", |
| "behavior", |
| "motion", |
| "control", |
| "actuators", |
| ]; |
|
|
| export const SAFETY_ORDER: StageId[] = ["supervisor", "health", "fallback"]; |
|
|
| export const LEARNING_ORDER: StageId[] = [ |
| "capture", |
| "mining", |
| "labeling", |
| "training", |
| "evaluation", |
| "simulation", |
| "release", |
| ]; |
|
|
| export const FLOW_ORDER: StageId[] = [ |
| "mission", |
| ...RUNTIME_ORDER, |
| "world", |
| ...SAFETY_ORDER, |
| ...LEARNING_ORDER, |
| ]; |
|
|
| export const STAGES: Record<StageId, Stage> = { |
| mission: { |
| id: "mission", |
| title: "Mission / Operator Intent", |
| plane: "ops", |
| role: "Where are we going and under what rules?", |
| detail: |
| "The autonomy application layer. A human or a fleet system sets the goal, the operational design domain, and the constraints the stack must respect. Route planning turns 'go to the depot' into a road-level sequence before behavior planning ever runs.", |
| inputs: ["Destination or task", "Operational design domain", "Fleet constraints"], |
| outputs: ["Route (road-level)", "Mission constraints", "Remote assistance requests"], |
| mlRole: "Usually classical search (A*, contraction hierarchies) over a road graph.", |
| failureModes: [ |
| "Route leaves the validated operating domain", |
| "Stale map data", |
| "No safe stopping point on the assigned route", |
| ], |
| keyIdea: "Autonomy is goal-directed. Without this layer the stack has nothing to optimize toward.", |
| }, |
| sensors: { |
| id: "sensors", |
| title: "Sensor Layer", |
| plane: "physical", |
| role: "Measure the physical world.", |
| detail: |
| "Cameras, LiDAR, radar, ultrasonics, IMU, GNSS, wheel odometry, and the vehicle CAN bus. Each has a different rate, latency, failure mode, and weather envelope — which is exactly why you carry more than one.", |
| inputs: ["Photons, radio returns, inertial forces"], |
| outputs: ["Raw frames, point clouds, radar returns, ego motion"], |
| mlRole: "None here. This is drivers, timing, and physics.", |
| failureModes: [ |
| "Lens occlusion (mud, sun, spray)", |
| "LiDAR return loss in heavy rain", |
| "GNSS multipath in urban canyons", |
| "Clock drift between sensors", |
| ], |
| keyIdea: |
| "Time synchronization and calibration belong here. A perfectly good detector on badly timestamped data produces garbage.", |
| }, |
| fusion: { |
| id: "fusion", |
| title: "Sensor Fusion", |
| plane: "perception", |
| role: "Put every sensor into one frame of reference and one clock.", |
| detail: |
| "Spatial calibration (extrinsics/intrinsics), temporal alignment, and association across modalities. Camera gives semantics and range ambiguity; LiDAR gives geometry; radar gives velocity through weather. Fusion decides which to trust where.", |
| inputs: ["Raw sensor streams", "Calibration", "Ego motion"], |
| outputs: ["Aligned multi-modal tensor / associated measurements"], |
| mlRole: |
| "Increasingly learned. Early-fusion networks consume raw modalities jointly rather than merging discrete detections late.", |
| failureModes: [ |
| "Extrinsic drift after a curb strike", |
| "Association errors (one object seen as two)", |
| "Latency skew between modalities at speed", |
| ], |
| keyIdea: |
| "Late fusion merges opinions; early fusion merges evidence. Modern stacks lean toward evidence.", |
| }, |
| perception: { |
| id: "perception", |
| title: "Perception / World Model", |
| plane: "perception", |
| role: "What is there?", |
| detail: |
| "Detection → tracking → classification → geometry → occupancy → free space → scene understanding. The output is a world model: ego state, tracked objects with uncertainty, lanes, boundaries, traffic lights, signs, and drivable area.", |
| inputs: ["Fused sensor data", "HD map priors (optional)"], |
| outputs: [ |
| "Ego state (pose, velocity, uncertainty)", |
| "Tracked objects (id, pose, velocity, class, confidence)", |
| "Road graph: lanes, boundaries, lights, signs", |
| "Occupancy / free space grid", |
| ], |
| mlRole: "Heavy. Neural networks and increasingly foundation-scale models for occupancy and latent world representations.", |
| failureModes: [ |
| "False negative on an unusual object class", |
| "Track ID switches through occlusion", |
| "Over-confident detections at long range", |
| ], |
| keyIdea: |
| "Occupancy representations matter because the world contains obstacles your object taxonomy never named.", |
| }, |
| prediction: { |
| id: "prediction", |
| title: "Prediction / Forecasting", |
| plane: "prediction", |
| role: "What is it likely to do?", |
| detail: |
| "For every agent, forecast plausible futures with probabilities — not a single trajectory. A pedestrian at a curb might cross (55%), stop (30%), or turn back (15%). Planning must stay safe across all plausible modes, not just the argmax.", |
| inputs: ["World model", "Map topology", "Agent history"], |
| outputs: ["Multimodal trajectory sets with probabilities", "Interaction-aware futures"], |
| mlRole: "Multimodal trajectory models; increasingly joint (scene-level) rather than per-agent.", |
| failureModes: [ |
| "Mode collapse to a single average future", |
| "Ignoring interaction (agents react to the ego vehicle)", |
| "Miscalibrated probabilities", |
| ], |
| keyIdea: |
| "Collapsing to one future is the classic failure. Safety lives in the tail modes.", |
| }, |
| behavior: { |
| id: "behavior", |
| title: "Behavior Planning", |
| plane: "planning", |
| role: "What maneuver should we commit to?", |
| detail: |
| "Discrete decisions: lane change, merge, yield, pull over, negotiate right-of-way. This layer reasons about rules, social norms, and risk — collision risk, uncertainty budget, and whether a fallback exists.", |
| inputs: ["Predicted futures", "Route", "Traffic rules"], |
| outputs: ["Maneuver decision + constraints for motion planning"], |
| mlRole: "Learned policies (imitation, RL) are common, usually fenced by rule-based checks.", |
| failureModes: [ |
| "Indecision oscillation (commit / abort / commit)", |
| "Overly conservative freezing in dense traffic", |
| "Right-of-way misjudgment", |
| ], |
| keyIdea: |
| "A good behavior layer commits. Flip-flopping is both unsafe and unreadable to other drivers.", |
| }, |
| motion: { |
| id: "motion", |
| title: "Motion Planning", |
| plane: "planning", |
| role: "Turn a maneuver into a physically executable trajectory.", |
| detail: |
| "Generate candidate trajectories → score → optimize → select. Constraints: vehicle dynamics, comfort (jerk limits), safety margins, obstacles, and traffic rules. Output is x(t), y(t), v(t), a(t), heading(t).", |
| inputs: ["Maneuver decision", "Occupancy + predictions", "Dynamic limits"], |
| outputs: ["Time-parameterized trajectory"], |
| mlRole: "Learned cost functions and learned trajectory proposals, with classical optimization on top.", |
| failureModes: [ |
| "Infeasible trajectory given tire/steering limits", |
| "Comfort/safety cost weights misbalanced", |
| "Planning horizon too short for the speed", |
| ], |
| keyIdea: |
| "The planner does not output steering. It outputs an intent in space and time; control makes it real.", |
| }, |
| control: { |
| id: "control", |
| title: "Control", |
| plane: "planning", |
| role: "Track the trajectory with actual actuator commands.", |
| detail: |
| "Trajectory tracking (PID, MPC, pure pursuit) converts the desired path into steering angle, brake, and throttle. Then it measures the actual state and corrects. This is the closed loop that makes autonomy a control system rather than a prediction system.", |
| inputs: ["Desired trajectory", "Measured ego state"], |
| outputs: ["steering, brake, throttle commands"], |
| mlRole: "Usually classical. Learned control appears mostly in research or low-level tuning.", |
| failureModes: [ |
| "Tracking error growth on low friction", |
| "Actuator saturation", |
| "Latency-induced instability", |
| ], |
| keyIdea: |
| "AI → action is wrong. It is AI → action → observe result → correct action.", |
| }, |
| actuators: { |
| id: "actuators", |
| title: "Actuators / Vehicle", |
| plane: "physical", |
| role: "Apply force to the world.", |
| detail: |
| "Steering, brake, and throttle actuators with their own limits, latencies, and redundancy requirements. The vehicle's response changes the world state, which the sensors then measure — closing the loop.", |
| inputs: ["Control commands"], |
| outputs: ["Physical motion", "Actuator health telemetry"], |
| mlRole: "None. This is mechanical engineering and functional safety.", |
| failureModes: [ |
| "Actuator fault or degraded authority", |
| "Command rejected by the vehicle platform", |
| "Redundancy path untested", |
| ], |
| keyIdea: "Redundant actuation is what makes a minimal-risk maneuver possible at all.", |
| }, |
| world: { |
| id: "world", |
| title: "Physical World", |
| plane: "physical", |
| role: "The ground truth nobody has direct access to.", |
| detail: |
| "The vehicle's motion changes the scene; other agents react; the sensors sample it again. Every loop through the stack is an estimate of a world that moved while you were computing.", |
| inputs: ["Vehicle motion", "Other agents", "Weather, light, road surface"], |
| outputs: ["New sensor observations"], |
| mlRole: "Simulated versions of this are increasingly generative models.", |
| failureModes: ["Distribution shift: the world stops resembling the training data"], |
| keyIdea: "Latency is a safety property. The world does not pause for your pipeline.", |
| }, |
| supervisor: { |
| id: "supervisor", |
| title: "Safety Supervisor", |
| plane: "safety", |
| role: "Can veto the planner.", |
| detail: |
| "Sits between the proposed trajectory and the actuators. Independent collision checking, geofence enforcement, actuator limits, and runtime invariants. If the proposal is unacceptable it overrides with a minimal-risk action rather than negotiating.", |
| inputs: ["Proposed trajectory", "Occupancy", "System health"], |
| outputs: ["SAFE → execute, or UNSAFE → override"], |
| mlRole: |
| "Deliberately deterministic. You do not want a learned component to be the last line of defense.", |
| failureModes: [ |
| "Over-triggering causes phantom braking", |
| "Under-triggering defeats the purpose", |
| "Shares a failure mode with the planner it is checking", |
| ], |
| keyIdea: |
| "The supervisor must be independent of the planner. Shared assumptions mean shared blind spots.", |
| }, |
| health: { |
| id: "health", |
| title: "System Health Monitoring", |
| plane: "safety", |
| role: "Is the stack itself trustworthy right now?", |
| detail: |
| "Sensor health, compute health, localization confidence, perception confidence, planning validity, and end-to-end latency budgets. Degradation here narrows the allowed operating envelope before anything visibly goes wrong.", |
| inputs: ["Per-module heartbeats, confidences, timing"], |
| outputs: ["Degradation level, envelope restrictions"], |
| mlRole: "Some learned anomaly detection, but mostly explicit invariants and watchdogs.", |
| failureModes: [ |
| "Silent degradation (module reports healthy while wrong)", |
| "Alarm fatigue from noisy thresholds", |
| ], |
| keyIdea: "Knowing you are uncertain is a capability, not an admission of weakness.", |
| }, |
| fallback: { |
| id: "fallback", |
| title: "Minimal-Risk Maneuver", |
| plane: "safety", |
| role: "The action taken when continuing is not acceptable.", |
| detail: |
| "Controlled deceleration, lane-keep to a stop, pull to the shoulder, or hand back to a remote operator. The fallback must be reachable with degraded sensing and degraded actuation — that is the whole point.", |
| inputs: ["Override signal", "Degradation level"], |
| outputs: ["Safe stop or handover"], |
| mlRole: "None. Rule-based and validated.", |
| failureModes: [ |
| "Stopping in a live lane", |
| "Fallback requires the sensor that just failed", |
| ], |
| keyIdea: "A fallback that depends on the failed component is not a fallback.", |
| }, |
| capture: { |
| id: "capture", |
| title: "Data Collection", |
| plane: "learning", |
| role: "Record what happened, especially when it was interesting.", |
| detail: |
| "Triggered capture around disengagements, overrides, near-misses, high-uncertainty perception, and rare classes. Storing everything is expensive and mostly useless; storing the right seconds is the skill.", |
| inputs: ["Runtime telemetry", "Safety override events"], |
| outputs: ["Event clips with full sensor context"], |
| mlRole: "Uncertainty-driven triggers are often model-based.", |
| failureModes: ["Triggers miss the failure", "Captured window too short for context"], |
| keyIdea: "The safety override is the highest-signal training label you get for free.", |
| }, |
| mining: { |
| id: "mining", |
| title: "Scenario Mining", |
| plane: "learning", |
| role: "Find every other time this nearly happened.", |
| detail: |
| "Search the fleet archive for scenarios similar to a captured event — same geometry, same agent behavior, same lighting. One incident becomes a regression suite of hundreds of cases.", |
| inputs: ["Event clips", "Fleet archive"], |
| outputs: ["Scenario cohort"], |
| mlRole: "Embedding search and ML-based scenario discovery.", |
| failureModes: ["Query too narrow (misses variants)", "Cohort dominated by one location"], |
| keyIdea: "One failure is an anecdote. A mined cohort is a benchmark.", |
| }, |
| labeling: { |
| id: "labeling", |
| title: "Labeling / QA", |
| plane: "learning", |
| role: "Establish ground truth.", |
| detail: |
| "Human and auto-labeling with a quality gate. Offline models with full future context (and no latency budget) can label better than the online stack ever could — that asymmetry is the engine of supervision.", |
| inputs: ["Scenario cohort"], |
| outputs: ["Ground-truth annotations"], |
| mlRole: "Auto-labeling with large offline models, audited by humans.", |
| failureModes: ["Label noise on exactly the hard cases", "Taxonomy drift between batches"], |
| keyIdea: "Offline models can see the future. Use that to teach the online model.", |
| }, |
| training: { |
| id: "training", |
| title: "Training", |
| plane: "learning", |
| role: "Update the learned components.", |
| detail: |
| "Retrain or fine-tune perception, prediction, and learned planning components on the enriched dataset. Reproducibility, dataset versioning, and rollback matter as much as accuracy.", |
| inputs: ["Labeled cohort", "Existing dataset"], |
| outputs: ["Candidate model"], |
| mlRole: "The obvious one — but it is a small slice of the loop.", |
| failureModes: ["Fixing one cohort regresses another", "Overfitting to mined events"], |
| keyIdea: "Training is cheap. Knowing whether the new model is better is the hard part.", |
| }, |
| evaluation: { |
| id: "evaluation", |
| title: "Evaluation", |
| plane: "learning", |
| role: "Is the candidate actually better?", |
| detail: |
| "Offline metrics plus closed-loop scoring on the regression suite. A model that improves mean average precision but degrades the cut-in scenario cohort does not ship.", |
| inputs: ["Candidate model", "Regression suite"], |
| outputs: ["PASS → release, FAIL → investigate"], |
| mlRole: "Learned metrics and failure prediction, plus hard scenario gates.", |
| failureModes: ["Metric–behavior mismatch", "Evaluation set leaks into training"], |
| keyIdea: "PASS/FAIL here is the real product decision.", |
| }, |
| simulation: { |
| id: "simulation", |
| title: "Simulation", |
| plane: "learning", |
| role: "Test the cases you cannot safely drive.", |
| detail: |
| "Replay captured logs against the new stack, then perturb them — earlier pedestrian, wetter road, later reaction. Closed-loop simulation catches the compounding errors that log replay alone hides.", |
| inputs: ["Scenario cohort", "Candidate stack"], |
| outputs: ["Closed-loop outcomes at scale"], |
| mlRole: "Generative and learned environments; sensor simulation.", |
| failureModes: ["Sim-to-real gap in sensor realism", "Agents in sim do not react to ego"], |
| keyIdea: "Log replay tests perception. Only closed-loop sim tests the stack.", |
| }, |
| release: { |
| id: "release", |
| title: "Release / OTA", |
| plane: "learning", |
| role: "Ship it — carefully.", |
| detail: |
| "Staged rollout, shadow mode, canary fleet, and a rollback path. The new stack runs alongside the old one and is compared on live data before it is given authority.", |
| inputs: ["Validated candidate"], |
| outputs: ["Deployed runtime", "Shadow-mode comparisons"], |
| mlRole: "None directly; this is release engineering.", |
| failureModes: ["No rollback path", "Fleet-wide simultaneous deploy"], |
| keyIdea: "Shadow mode gives you evidence without giving the new model control.", |
| }, |
| }; |
|
|
| export type Frame = { |
| label: string; |
| headline: string; |
| safety: "nominal" | "override"; |
| capture?: string; |
| readouts: Partial<Record<StageId, string[]>>; |
| }; |
|
|
| export const FRAMES: Frame[] = [ |
| { |
| label: "t + 0.0s", |
| headline: "Cruising toward a signalized crosswalk at 11 m/s.", |
| safety: "nominal", |
| readouts: { |
| sensors: ["camera 30 Hz · lidar 10 Hz · radar 20 Hz", "GNSS fix: RTK, 3 cm"], |
| fusion: ["extrinsics nominal", "time skew 4 ms"], |
| perception: ["veh #123 @ 42 m, 10.8 m/s", "ped #42 @ curb, 0.2 m/s", "light: GREEN"], |
| prediction: ["ped #42 — wait 0.82 / cross 0.14 / turn back 0.04"], |
| behavior: ["maintain lane · monitor crosswalk"], |
| motion: ["v_target 11.0 m/s · jerk 0.4 m/s³"], |
| control: ["steer +0.2° · brake 0% · throttle 12%"], |
| actuators: ["all channels nominal"], |
| supervisor: ["collision check: clear (TTC 6.1 s)"], |
| health: ["all modules green · latency 84 ms"], |
| }, |
| }, |
| { |
| label: "t + 1.4s", |
| headline: "Pedestrian steps off the curb. Prediction flips to multimodal.", |
| safety: "nominal", |
| readouts: { |
| sensors: ["lidar returns on ped torso +18 pts"], |
| fusion: ["radar doppler confirms lateral motion"], |
| perception: ["ped #42 @ 1.1 m into lane, 1.4 m/s", "confidence 0.94"], |
| prediction: ["ped #42 — cross 0.55 / stop 0.30 / retreat 0.15", "mode spread widening"], |
| behavior: ["yield to pedestrian · abort lane hold speed"], |
| motion: ["decel profile −2.1 m/s² · stop line 18 m"], |
| control: ["steer 0.0° · brake 22% · throttle 0%"], |
| actuators: ["brake pressure rising"], |
| supervisor: ["TTC 2.4 s · within margin"], |
| health: ["all modules green · latency 91 ms"], |
| }, |
| }, |
| { |
| label: "t + 2.1s", |
| headline: "Occlusion behind a parked van. Supervisor overrides the planner.", |
| safety: "override", |
| capture: "Safety override — occluded pedestrian, urban crosswalk", |
| readouts: { |
| sensors: ["camera occluded 34% · lidar shadow behind van"], |
| fusion: ["association gap on ped #42 for 2 frames"], |
| perception: ["ped #42 track COASTING (no measurement)", "occupancy: unobserved cell in path"], |
| prediction: ["ped #42 — cross 0.61 / stop 0.24 / retreat 0.15", "uncertainty above budget"], |
| behavior: ["proposal: creep forward 3 m"], |
| motion: ["trajectory enters unobserved cell"], |
| control: ["OVERRIDDEN by supervisor"], |
| actuators: ["brake 68% · commanded by fallback"], |
| supervisor: ["UNSAFE — path crosses unobserved occupancy", "override issued in 12 ms"], |
| health: ["perception confidence degraded → envelope narrowed"], |
| fallback: ["comfortable stop, hold at stop line"], |
| }, |
| }, |
| { |
| label: "t + 4.6s", |
| headline: "Pedestrian clears. Event is queued to the learning plane.", |
| safety: "nominal", |
| readouts: { |
| sensors: ["occlusion cleared"], |
| fusion: ["ped #42 re-associated to same track id"], |
| perception: ["crosswalk clear · occupancy observed"], |
| prediction: ["no agent conflicts in 5 s horizon"], |
| behavior: ["resume · proceed through crosswalk"], |
| motion: ["accel profile +1.2 m/s² · v_target 11.0 m/s"], |
| control: ["steer −0.1° · brake 0% · throttle 9%"], |
| actuators: ["nominal"], |
| supervisor: ["returned authority to planner"], |
| health: ["all modules green"], |
| capture: ["clip saved: −8 s to +6 s, all modalities"], |
| }, |
| }, |
| ]; |
|
|
