/** * Central glossary: every metric, status, and concept shown anywhere in the UI * is defined once here and rendered identically everywhere via / * (src/components/help/InfoTip.tsx) and the global help menu. * * Entry fields: * term display name * short one-line definition (used in dense tooltips) * detail how it is computed / what it means operationally * caveat honesty note: limits, approximations, failure modes (optional) */ export interface GlossaryEntry { term: string; short: string; detail: string; caveat?: string; /** Category used by the glossary browser in the help menu. */ category: GlossaryCategory; } export type GlossaryCategory = | 'Detection metrics' | 'Geometry & LiDAR' | 'Tracking' | 'Anomaly & rarity' | 'Safety (SSAM)' | 'Human grading & sampling' | 'Aggregation & scale' | 'Pipeline & statuses' | 'Failure reasons' | 'Ground truth' | 'Operations' | 'EM readiness' | 'Hardware acceleration' | 'Safety & compliance' | 'Sequential testing' | 'Perception fusion' | 'Retrospective safety' | 'Closed-loop evaluation' | 'Launch readiness' | 'Release governance' | 'Production hardening' | 'ROTR (right-of-the-road)'; export const GLOSSARY: Record = { // ------------------------------------------------------------ detection metrics precision: { term: 'Precision', category: 'Detection metrics', short: 'Of all labels the system produced, the fraction that are correct.', detail: 'TP / (TP + FP). A true positive is a predicted box matched to a reference object at 3D IoU ≥ 0.5. High precision means few phantom / wrong labels.', caveat: 'Only as trustworthy as the reference: against pseudo-GT, "correct" means "agrees with the auto-label consensus", not human truth.', }, recall: { term: 'Recall', category: 'Detection metrics', short: 'Of all real objects present, the fraction the system labeled.', detail: 'TP / (TP + FN). A false negative is a reference object with no matching predicted box at 3D IoU ≥ 0.5. Low recall means missed objects — the most dangerous failure mode for AV perception.', caveat: 'Recall against pseudo-GT under-counts misses that the auto-labeler also failed to see.', }, safety_recall: { term: 'Safety recall', category: 'Detection metrics', short: 'Recall computed only over safety-critical objects.', detail: 'Same TP/(TP+FN) formula, restricted to objects flagged safety-critical (vulnerable road users, near-path objects, low-TTC actors). Held to a stricter promotion threshold than headline recall.', caveat: 'Small denominators: per-cohort safety recall can swing on a handful of objects — check n before reacting.', }, f1: { term: 'F1', category: 'Detection metrics', short: 'Harmonic mean of precision and recall.', detail: '2·P·R / (P + R). A single balance number; useful for ranking but hides which side (misses vs phantoms) is failing — always drill into P and R separately.', }, map_3d: { term: 'mAP (3D)', category: 'Detection metrics', short: 'Mean Average Precision over classes using 3D IoU matching.', detail: 'For each class, average precision is the area under the precision–recall curve sweeping the confidence threshold; mAP is the mean across classes. Matching uses 3D IoU ≥ 0.5.', caveat: 'Sensitive to class balance: rare classes (motorcycle, bus) move mAP as much as vehicle despite far fewer instances.', }, rare_recall: { term: 'Rare recall', category: 'Detection metrics', short: 'Recall restricted to rare-event objects.', detail: 'Recall over objects in scenarios tagged rare (near-miss, extreme TTC, unusual classes / poses). The main target metric of the rare-event mining flywheel.', }, confidence: { term: 'Confidence', category: 'Detection metrics', short: 'The model\u2019s own probability that a predicted label is correct.', detail: 'Raw score in [0, 1] emitted by the detection head. Used by the quality gate (min_confidence) and by review sampling to build the low/mid/high-confidence strata.', caveat: 'Calibrated only approximately: a 0.9 does not mean 90% empirical correctness in every cohort.', }, anomaly_rate: { term: 'Anomaly rate', category: 'Detection metrics', short: 'Fraction of labels flagged anomalous by the ensemble detectors.', detail: 'Count of objects whose anomaly score exceeds the policy threshold, divided by all evaluated objects.', }, // ------------------------------------------------------------ geometry & LiDAR iou_3d: { term: '3D IoU', category: 'Geometry & LiDAR', short: 'Volume overlap between predicted and reference 3D boxes.', detail: 'Intersection volume / union volume of the two oriented boxes, in [0, 1]. IoU ≥ 0.5 counts as a match; mean IoU over matched pairs measures how tightly boxes fit.', caveat: 'A box can match at IoU 0.5 and still have significant heading or size error — see orientation / dimension error.', }, position_error: { term: 'Position error', category: 'Geometry & LiDAR', short: 'Distance in meters between predicted and reference box centers.', detail: 'Euclidean distance between 3D box centroids. Gated per quality policy (default max 0.5 m); large values indicate localization drift.', }, orientation_error: { term: 'Orientation error', category: 'Geometry & LiDAR', short: 'Heading angle difference in degrees between predicted and reference box.', detail: 'Absolute yaw difference, wrapped to [0°, 180°]. Errors near 180° usually mean a flipped heading, which corrupts downstream velocity estimates.', }, dimension_error: { term: 'Dimension error', category: 'Geometry & LiDAR', short: 'How far box length/width/height deviate from the class-typical size.', detail: 'Relative deviation of box dimensions vs the class prior (e.g. a “pedestrian” box 3 m wide fails). Used as a geometric validation check.', }, point_density: { term: 'Point density', category: 'Geometry & LiDAR', short: 'LiDAR points per cubic meter inside the labeled box.', detail: 'Points-in-box divided by box volume. Very low density means the label rests on little sensor evidence — common at long range or under occlusion.', }, points_in_box: { term: 'Points in box', category: 'Geometry & LiDAR', short: 'Absolute number of LiDAR returns inside the 3D box.', detail: 'Boxes with fewer points than the policy minimum are flagged INSUFFICIENT_POINT_SUPPORT: there is not enough evidence to verify the object geometrically.', }, point_in_box_ratio: { term: 'Point-in-box ratio (occupancy)', category: 'Geometry & LiDAR', short: 'Fraction of the box volume actually supported by LiDAR returns.', detail: 'Occupancy of the box by points relative to expectation for the class. Low occupancy with a large box suggests the box is oversized or misplaced.', }, ground_contact: { term: 'Ground contact', category: 'Geometry & LiDAR', short: 'Whether the box bottom sits plausibly on the road surface.', detail: 'Vertical distance from the box bottom face to the estimated ground plane. Floating or buried boxes fail geometric validation.', }, sensor_consistency: { term: 'Sensor consistency', category: 'Geometry & LiDAR', short: 'Agreement between camera and LiDAR evidence for the same object.', detail: 'The 3D box is projected into the camera image and compared to 2D detections. Disagreement (object in LiDAR but not camera, or vice versa) raises SENSOR_DISAGREEMENT.', caveat: 'Expected to degrade in rain/fog/night-glare; disagreement there is a weaker error signal.', }, // ------------------------------------------------------------ tracking idf1: { term: 'IDF1', category: 'Tracking', short: 'F1 over identity assignments: how consistently the same object keeps the same track ID.', detail: 'Matches predicted and reference trajectories, then computes F1 over correctly-identified detections. Penalizes both ID swaps and missed coverage.', }, id_switch: { term: 'ID switch', category: 'Tracking', short: 'A track changed identity mid-trajectory.', detail: 'The same physical object was assigned a new track ID (or two objects swapped IDs). Corrupts velocity history and intent prediction; gated as a hard failure.', }, fragmentation: { term: 'Track fragmentation', category: 'Tracking', short: 'One object\u2019s trajectory broken into multiple short tracks.', detail: 'Count (or rate) of interruptions where tracking lost then re-acquired the object. High fragmentation usually accompanies occlusion or low point density.', }, track_quality: { term: 'Track quality', category: 'Tracking', short: 'Composite 0–1 score of trajectory smoothness and consistency.', detail: 'Combines fragmentation, ID stability and kinematic plausibility (no teleports, physical accelerations). Below policy threshold ⇒ TRACK_FRAGMENTATION flag.', }, // ------------------------------------------------------------ anomaly & rarity anomaly_score: { term: 'Anomaly score', category: 'Anomaly & rarity', short: 'Ensemble estimate (0–1) of how unusual this label is vs the population.', detail: 'Score fused from KNN distance, Local Outlier Factor, Isolation Forest, One-Class SVM and DBSCAN outlier flags over geometric + contextual features. Above the policy threshold ⇒ flagged ANOMALY.', caveat: 'Anomalous ≠ wrong: genuinely rare-but-correct labels also score high — that is exactly what rare-event mining exploits.', }, ensemble_strategy: { term: 'Ensemble strategy', category: 'Anomaly & rarity', short: 'How the individual detector scores are fused into one anomaly score.', detail: 'max = most sensitive (any detector can flag); mean = balanced; vote = majority of detectors must agree (most conservative). Choose based on tolerance for false alarms vs missed anomalies.', }, rarity_score: { term: 'Rarity score', category: 'Anomaly & rarity', short: 'How under-represented this sample\u2019s scenario is in the training corpus.', detail: 'Inverse frequency of the sample\u2019s cohort (class × weather × lighting × scenario) in existing training data, blended with the anomaly score. High rarity ⇒ high value for training.', }, haystack: { term: 'Needle-in-haystack view', category: 'Anomaly & rarity', short: 'Projection of the whole population where rare events light up against the nominal mass.', detail: '2D embedding (feature projection) of labels: gray points are nominal, colored points are anomaly-flagged / rare candidates. Clusters of colored points suggest a systematic blind spot, not a one-off.', }, // ------------------------------------------------------------ safety (SSAM) ttc: { term: 'TTC (Time to Collision)', category: 'Safety (SSAM)', short: 'Seconds until two road users would collide on current trajectories.', detail: 'Computed from relative position and velocity at the conflict point. Lower is more dangerous; conflicts under ~1.5 s are typically classified critical.', }, pet: { term: 'PET (Post-Encroachment Time)', category: 'Safety (SSAM)', short: 'Gap in seconds between one road user leaving a conflict area and another entering it.', detail: 'Unlike TTC it does not require projected collision courses — it measures how narrowly paths actually missed each other. Small PET = near miss.', }, severity_index: { term: 'Severity index', category: 'Safety (SSAM)', short: 'Composite ranking of how dangerous a surrogate-safety conflict is.', detail: 'Combines TTC/PET, speeds, masses/classes of participants and conflict geometry into one score used to rank intersections and events (critical / high / medium / low).', }, // ------------------------------------------------------------ human grading & sampling grader_consensus: { term: 'Grader consensus', category: 'Human grading & sampling', short: 'Agreement level among independent graders of the same label.', detail: 'Fraction of graders agreeing with the majority verdict. Below the policy threshold ⇒ GRADER_DISAGREEMENT and the label routes to senior review.', }, cohens_kappa: { term: "Cohen's Kappa", category: 'Human grading & sampling', short: 'Agreement between two graders, corrected for chance.', detail: '(observed agreement − chance agreement) / (1 − chance agreement). 0 = no better than chance, 1 = perfect. Rule of thumb: > 0.6 substantial, > 0.8 near-perfect.', caveat: 'Only defined for exactly two raters; distorted when one verdict dominates (prevalence problem).', }, fleiss_kappa: { term: "Fleiss' Kappa", category: 'Human grading & sampling', short: 'Chance-corrected agreement generalized to many graders.', detail: 'Extends Cohen\u2019s Kappa to a fixed panel of ≥ 3 raters rating each item. Used for the grader-pool health metrics.', }, krippendorff_alpha: { term: "Krippendorff's Alpha", category: 'Human grading & sampling', short: 'The most general agreement coefficient: any #raters, missing data, any scale.', detail: 'Based on observed vs expected disagreement; handles graders who skip items. Preferred when the grading matrix is sparse. α ≥ 0.8 is the conventional reliability bar.', }, wilson_ci: { term: 'Wilson 95% CI', category: 'Human grading & sampling', short: 'Confidence interval for a proportion that stays honest at small n and extreme p.', detail: 'Wilson score interval: solves the normal approximation for the true proportion instead of centering on the sample value. Unlike the naive ±1.96·SE interval it never leaves [0, 1] and behaves at p ≈ 0 or 1.', caveat: 'A 95% CI means: under repeated sampling, ~95% of such intervals cover the truth — not "95% probability" for this one interval.', }, stratified_sampling: { term: 'Stratified risk-weighted sampling', category: 'Human grading & sampling', short: 'Review budget split across risk strata so rare, risky slices are measured, not just the easy mass.', detail: 'The population is partitioned into strata (e.g. low/mid/high confidence × safety-critical); each stratum gets reviews proportional to size × risk weight (Neyman-style). Per-stratum Wilson CIs combine into one population estimate.', caveat: 'Estimates are unbiased only if reviews within each stratum are randomly drawn — never cherry-pick inside a stratum.', }, review_sample: { term: 'Review sample', category: 'Human grading & sampling', short: 'The statistically selected subset of labels actually sent to human reviewers.', detail: 'Drawn from the candidate pool by stratified risk-weighted sampling. Its verdicts calibrate the automated headline metrics with confidence intervals — reviewing everything at this scale is impossible.', }, candidate_pool: { term: 'Candidate pool', category: 'Human grading & sampling', short: 'All labels eligible for human review, ranked by risk before sampling.', detail: 'Union of error-index hits, low-confidence labels, anomalies and safety-critical objects. The sample is drawn from this pool, not from the raw population.', }, sampling_verified: { term: 'Sampling-verified', category: 'Human grading & sampling', short: 'This number was calibrated by human review of a statistical sample.', detail: 'The headline (automated) value is shown alongside a human-verified estimate with a 95% CI, derived from stratified review. Where they disagree, trust the interval.', }, // ------------------------------------------------------------ aggregation & scale metric_cube: { term: 'Metric cube', category: 'Aggregation & scale', short: 'Pre-aggregated sufficient statistics (TP/FP/FN/IoU sums…) per dimension combination.', detail: 'During an evaluation run each partition emits partial counts per (class, weather, lighting, road type, scenario, sensor, distance, speed, occlusion) cell; the reduce step materializes the cube. Any filter/group-by query then reads cell sums instead of scanning records — that is why dashboard queries take milliseconds over 300k+ objects.', caveat: 'Cube answers are exact for additive stats (counts, sums). Non-additive stats (quantiles, distinct counts) come from sketches and are approximate.', }, query_source: { term: 'Query source (cache / cube / scan)', category: 'Aggregation & scale', short: 'Where this panel\u2019s numbers came from, and how fast.', detail: 'cache = identical query answered from the in-memory result cache (~0.1 ms). cube = aggregated from the metric cube (~10 ms). scan = fell back to reading raw partitions (slow; only for forensic drill-down).', }, hll: { term: 'HyperLogLog (HLL)', category: 'Aggregation & scale', short: 'Approximate distinct-count sketch: huge cardinalities in a few KB.', detail: 'Hashes each item and tracks maximum leading-zero patterns per register; the harmonic mean of registers estimates distinct count. Standard error ≈ 1.04/√m (≈ 2% here). Sketches merge losslessly across partitions.', caveat: 'Approximate by design — shown next to the exact count when both exist, and always labeled "approx".', }, quantile_sketch: { term: 'Quantile histogram (sketch)', category: 'Aggregation & scale', short: 'Compact histogram that answers percentile queries without keeping raw values.', detail: 'Fixed-bin histogram maintained per partition and merged at reduce time; p10/p50/p90 are interpolated from bin counts. Error is bounded by bin width.', caveat: 'Percentiles are approximate (bin-resolution); exact tails may differ slightly.', }, reservoir_sampling: { term: 'Reservoir sampling', category: 'Aggregation & scale', short: 'Keeps a fixed-size uniform random sample from a stream of unknown length.', detail: 'Each new item replaces a random slot with probability k/n. Used to retain representative exemplar objects per cohort for drill-down without storing everything.', }, approx_vs_exact: { term: 'Approx vs exact', category: 'Aggregation & scale', short: 'Every number is labeled by provenance: exact (cube/counts) or approx (sketch/sample-derived).', detail: 'Counts and additive aggregates are exact. Distinct counts (HLL), percentiles (quantile sketches) and human-review estimates (sampling + CI) are approximate, and the UI marks each with an "approx" tag plus this explanation.', }, cohort: { term: 'Cohort', category: 'Aggregation & scale', short: 'A population slice defined by dimension values, e.g. pedestrian × night × rain.', detail: 'Cohorts are the primary unit of analysis: metrics are compared across cohorts to find where the model fails, before ever looking at individual annotations.', }, container: { term: 'Container', category: 'Aggregation & scale', short: 'A physical grouping of annotations — one scene/segment captured together.', detail: 'Sits between cohort and annotation in the drill-down: Dataset → Cohort → Container → Annotation. Container-level aggregates (per-scene precision/recall/risk) localize failures to specific captures.', }, evaluation_run: { term: 'Evaluation run', category: 'Aggregation & scale', short: 'A first-class asynchronous job that evaluates one model version over one population.', detail: 'Lifecycle: created → queued → running (workers emit partial stats per partition) → reducing → materializing (cube + error index + sketches) → published. Only published runs are queryable.', }, population: { term: 'Population', category: 'Aggregation & scale', short: 'The immutable, versioned set of objects an evaluation run scores.', detail: 'Generated / ingested once, stored as partitioned columnar files, then referenced by ID. Immutability is what makes runs reproducible and comparable.', }, lineage: { term: 'Lineage', category: 'Aggregation & scale', short: 'The complete recipe to reproduce a run bit-for-bit.', detail: 'Dataset version, model version + checkpoint hash, label version, evaluator code version, metric version, threshold config, sampling config, seed, hardware, timestamp. Identical inputs + seed ⇒ identical outputs.', }, distribution_shift: { term: 'Distribution shift', category: 'Aggregation & scale', short: 'The evaluation population\u2019s cohort mix differs from what the model was trained on.', detail: 'Train vs eval share is compared per cohort; cohorts whose share grew materially AND whose recall lags the overall value are flagged — the model is being asked about conditions it rarely saw.', caveat: 'Shift is a risk signal, not proof of failure: confirm with the cohort\u2019s actual metrics.', }, risk_score: { term: 'Risk score', category: 'Aggregation & scale', short: 'Container-level priority: how likely this scene hides real, important errors.', detail: 'Blends error density (FN/FP counts), safety-critical presence, anomaly rate and low review coverage into a 0–1 rank used by the "Highest risk" sort and review candidate pool.', }, error_index: { term: 'Error index', category: 'Aggregation & scale', short: 'A dedicated index of every potential error, searchable by multiple criteria.', detail: 'Each FN / FP / localization / anomaly / low-confidence hit is indexed with its dimensions, severity and container, so "worst-N containers where class=pedestrian AND lighting=night" is a lookup, not a scan.', }, embedding_similarity: { term: 'Similarity search', category: 'Aggregation & scale', short: 'Finds containers structurally similar to a query container.', detail: 'Cosine similarity over 32-dimensional structural embeddings (dimension mix, error profile), hybridized with hard filters on scenario/lighting. Used to grow one found failure into its whole family.', caveat: 'Embeddings are compact projections — treat results as candidates to inspect, not proof of equivalence.', }, // ------------------------------------------------------------ pipeline & statuses quality_policy: { term: 'Quality policy', category: 'Pipeline & statuses', short: 'The versioned set of thresholds every label is gated against.', detail: 'Defines min IoU, max position/orientation error, min points-in-box, min confidence, min consensus, anomaly threshold, tracking requirements and regression blocking. Every triage decision records the policy ID that produced it.', }, quality_gate: { term: 'Quality gate', category: 'Pipeline & statuses', short: 'A single pass/fail check of one measurement against one policy threshold.', detail: 'Each gate line shows the measured value, the threshold, and the verdict. A label\u2019s triage status is the deterministic result of its gate lines — hover any decision to see them.', }, status_auto_graded: { term: 'AUTO_GRADED', category: 'Pipeline & statuses', short: 'Passed every applicable quality gate; verified without human involvement.', detail: 'The label met all policy thresholds (geometry, confidence, consensus, tracking, anomaly). Eligible for training data at the automated confidence tier.', }, status_flagged: { term: 'FLAGGED', category: 'Pipeline & statuses', short: 'Failed one or more gates; routed to human review.', detail: 'The failure reasons list which gates failed. Flagged labels enter the HITL queue ordered by severity and safety criticality.', }, status_verified: { term: 'VERIFIED', category: 'Pipeline & statuses', short: 'A human reviewer confirmed (or corrected) this label.', detail: 'Highest trust tier. Only VERIFIED labels enter training datasets; corrections flow back as new ground truth.', }, status_rejected: { term: 'REJECTED', category: 'Pipeline & statuses', short: 'A human reviewer judged the label wrong and it was discarded.', detail: 'Rejected labels are excluded from training and count against the auto-labeler\u2019s precision in audit reporting.', }, status_pending: { term: 'PENDING', category: 'Pipeline & statuses', short: 'Not yet evaluated — waiting in the pipeline queue.', detail: 'No triage decision exists yet for this label.', }, verification_rate: { term: 'Verification rate', category: 'Pipeline & statuses', short: 'Share of processed labels that reached VERIFIED status.', detail: 'VERIFIED / all triaged. Rises as HITL clears flagged items and as auto-grading improves.', }, automation_rate: { term: 'Automation rate', category: 'Pipeline & statuses', short: 'Share of labels fully handled without any human touch.', detail: 'AUTO_GRADED / all triaged. The core economic metric: every point of automation is human review budget freed for the hard cases.', }, do_not_promote: { term: 'DO NOT PROMOTE', category: 'Pipeline & statuses', short: 'The candidate model failed the promotion policy against the baseline.', detail: 'Promotion compares headline and safety metrics with policy-defined maximum allowed drops (e.g. recall −1 pt, safety recall −0.5 pt) and scans all cohorts for localized regressions. Any violated rule is listed with its measured delta and threshold.', }, regression_detected: { term: 'REGRESSION', category: 'Pipeline & statuses', short: 'A cohort where the candidate is significantly worse than the baseline.', detail: 'Cohort-level metric delta beyond the policy tolerance with sufficient n. The line shows cohort, metric, baseline → candidate values and the delta.', }, run_status: { term: 'Run status', category: 'Pipeline & statuses', short: 'Lifecycle stage of an evaluation run.', detail: 'created → queued → running (workers score partitions) → reducing (merging partial stats) → materializing (writing cube, error index, sketches) → published (queryable) or failed.', }, triage: { term: 'Automated triage', category: 'Pipeline & statuses', short: 'The deterministic decision layer that routes every label to auto-grade, flag or reject.', detail: 'Runs each label\u2019s evaluation evidence through the quality policy gates and records status + failure reasons + policy ID. Fully auditable: same evidence + policy ⇒ same decision.', }, process_units: { term: 'Process units', category: 'Operations', short: 'Normalized compute-cost currency consumed by pipeline stages.', detail: 'Each stage (labeling, evaluation, anomaly ensembles, training) reports its consumption in a common unit so cost per verified event / per million frames can be tracked and budgeted.', }, // ------------------------------------------------------------ failure reasons reason_low_iou: { term: 'LOW_IOU', category: 'Failure reasons', short: 'Box overlaps reference below the policy minimum 3D IoU.', detail: 'The label localizes the right object poorly (or the wrong object). See Position/Orientation error to understand which.', }, reason_position_error: { term: 'POSITION_ERROR', category: 'Failure reasons', short: 'Box center too far from the reference position.', detail: 'Exceeded max position error (m) in the quality policy.', }, reason_orientation_error: { term: 'ORIENTATION_ERROR', category: 'Failure reasons', short: 'Heading disagrees with reference beyond tolerance.', detail: 'Often a 180° flip; corrupts velocity and intent downstream.', }, reason_insufficient_point_support: { term: 'INSUFFICIENT_POINT_SUPPORT', category: 'Failure reasons', short: 'Too few LiDAR points inside the box to verify it.', detail: 'Below policy minimum points-in-box. The object may be real (far / occluded) but cannot be auto-verified.', }, reason_sensor_disagreement: { term: 'SENSOR_DISAGREEMENT', category: 'Failure reasons', short: 'Camera and LiDAR evidence conflict for this object.', detail: 'One modality sees the object, the other does not (projection mismatch). Routes to human review with both views attached.', }, reason_anomaly: { term: 'ANOMALY', category: 'Failure reasons', short: 'Ensemble anomaly score above the policy threshold.', detail: 'The label is statistically unusual vs the population. Could be an error or a genuinely rare event — that is what review decides.', }, reason_grader_disagreement: { term: 'GRADER_DISAGREEMENT', category: 'Failure reasons', short: 'Human graders disagreed below the consensus threshold.', detail: 'Routed to senior review; also feeds the grader-pool agreement (Kappa/Alpha) monitoring.', }, reason_id_switch: { term: 'ID_SWITCH', category: 'Failure reasons', short: 'Track identity changed mid-trajectory.', detail: 'Hard tracking failure; see Tracking metrics.', }, reason_track_fragmentation: { term: 'TRACK_FRAGMENTATION', category: 'Failure reasons', short: 'Trajectory broken into fragments or track quality below threshold.', detail: 'Frequently co-occurs with occlusion and low point density.', }, reason_low_confidence: { term: 'LOW_CONFIDENCE', category: 'Failure reasons', short: 'Model confidence below the policy minimum.', detail: 'Not necessarily wrong — low-confidence strata get elevated human sampling instead of automatic rejection.', }, reason_model_regression: { term: 'MODEL_REGRESSION', category: 'Failure reasons', short: 'The producing model version is regressed vs baseline for this slice.', detail: 'Labels from a regressed model are blocked from auto-grading when the policy sets block_on_model_regression.', }, // error index types error_tp: { term: 'TP (True Positive)', category: 'Failure reasons', short: 'A predicted label correctly matched to a real object.', detail: 'Predicted box matched a reference object at 3D IoU ≥ 0.5. The numerator of both precision and recall.', }, error_fn: { term: 'FN (False Negative)', category: 'Failure reasons', short: 'A reference object the model missed entirely.', detail: 'No predicted box matched this object at IoU ≥ 0.5. The dominant and most safety-relevant error class.', }, error_fp: { term: 'FP (False Positive)', category: 'Failure reasons', short: 'A predicted box with no corresponding real object.', detail: 'Phantom label. Inflates precision problems and can cause ghost braking downstream.', }, error_localization: { term: 'LOCALIZATION', category: 'Failure reasons', short: 'Object matched, but geometry is significantly off.', detail: 'IoU passed the match floor but position/orientation/size error breached policy.', }, error_low_conf: { term: 'LOW_CONF', category: 'Failure reasons', short: 'Correct-looking label with confidence below the review threshold.', detail: 'Indexed so review sampling can target the uncertain mass.', }, // ------------------------------------------------------------ ground truth gt_pseudo: { term: 'PSEUDO_GROUND_TRUTH', category: 'Ground truth', short: 'Reference derived from the auto-labeling consensus itself — not human truth.', detail: 'Generated by ensembling model outputs over time (offline tracking, multi-frame smoothing). Enables evaluation at scale but shares blind spots with the models that produced it.', caveat: 'Metrics against pseudo-GT are upper bounds: an error both systems make is invisible. Never promote a model on pseudo-GT numbers alone.', }, gt_vendor: { term: 'VENDOR_GROUND_TRUTH', category: 'Ground truth', short: 'Labels purchased from an external annotation vendor.', detail: 'Human-drawn but with variable QA depth; treated as medium-confidence reference. Vendor batches are spot-audited via the sampling pipeline.', }, gt_human_verified: { term: 'HUMAN_VERIFIED_GROUND_TRUTH', category: 'Ground truth', short: 'Labels confirmed or corrected by this platform\u2019s own HITL review.', detail: 'High-confidence reference: each label carries its reviewer trail and gate evidence.', }, gt_gold: { term: 'GOLD_STANDARD', category: 'Ground truth', short: 'Expert multi-pass annotations — the calibration reference.', detail: 'Small, expensive, maximum-quality set used to calibrate every other GT tier and the graders themselves.', }, gt_coverage: { term: 'GT coverage', category: 'Ground truth', short: 'Fraction of the dataset that has any reference ground truth.', detail: 'Labels without reference GT can only be checked by GT-free gates (geometry plausibility, sensor consistency, anomaly, consensus).', }, // ------------------------------------------------------------ operations sse: { term: 'Live stream (SSE)', category: 'Operations', short: 'This panel updates in real time over a server-sent-events stream.', detail: 'The backend pushes pipeline, queue and run-progress state every second on /api/events/stream; no refresh needed.', }, partition: { term: 'Partition', category: 'Operations', short: 'One shard of the population, evaluated independently by a worker.', detail: 'Workers emit partial sufficient statistics per partition; the reduce step merges them. Stand-in for distributed executors on this single-node deployment.', }, throughput: { term: 'Throughput', category: 'Operations', short: 'Objects evaluated per second across all active workers.', detail: 'Instantaneous rate from completed partitions; drives the ETA estimate.', }, query_cache: { term: 'Query cache', category: 'Operations', short: 'In-memory result cache for aggregate queries.', detail: 'Keyed by a hash of (run, filters, group-by, metrics). Hit rate shown in the header; identical dashboard queries return in ~0.1 ms.', caveat: 'Invalidated when a run publishes new artifacts.', }, // ------------------------------------------------- root cause analysis (RCA) psi: { term: 'PSI (Population Stability Index)', category: 'Aggregation & scale', short: 'Practical magnitude of a distribution change between two populations.', detail: 'Sum over bins of (actual% − expected%) × ln(actual%/expected%). Rule of thumb: <0.02 negligible, <0.1 small, <0.25 moderate, ≥0.25 large. Used in the Root Cause Lab to compare offline vs shadow populations independently of p-values.', caveat: 'At large n almost any change is statistically significant; PSI answers "does it matter", not "is it detectable".', }, effective_sample_size: { term: 'Effective sample size', category: 'Human grading & sampling', short: 'The number of independent observations your correlated data is actually worth.', detail: 'Rows from the same entity (scene/track) are correlated; n_eff = n / design-effect where the design effect grows with cluster size and intra-cluster correlation. Confidence intervals in the Root Cause Lab use n_eff, not raw row counts.', caveat: 'Using raw n on clustered evaluation data makes CIs look several times narrower than they really are.', }, simpsons_paradox: { term: "Simpson's paradox", category: 'Aggregation & scale', short: 'Aggregate trend reverses the trend present in every segment.', detail: 'If a model improves within every segment but the population mix shifts toward segments where it is weaker, the aggregate can show a regression that no segment shows. The conditional-performance heatmap in the Root Cause Lab surfaces this: consistent per-segment deltas + a mix shift.', }, training_serving_skew: { term: 'Training–serving skew', category: 'Operations', short: 'A feature is computed differently offline than in production serving.', detail: 'Unit changes, normalization differences or missing upstream signals make the serving-time feature distribution diverge from the offline one; the model then sees inputs unlike its training data. Detected in the feature-parity stage as a large standardized mean difference concentrated in specific features.', caveat: 'Diagnose within segments: an aggregate feature delta can also be an honest population shift.', }, shadow_evaluation: { term: 'Shadow evaluation', category: 'Operations', short: 'Running a candidate model on live traffic without acting on its output.', detail: 'The candidate scores real production inputs in parallel with the incumbent; outcomes are compared post-hoc. Sensitive to sampling bias, serving-config differences and provisional labels — which is why the Root Cause Lab audits the shadow pipeline before trusting its number.', }, costumed_pedestrian: { term: 'Costumed pedestrian', category: 'Anomaly & rarity', short: 'A person whose costume distorts the human silhouette a detector relies on.', detail: 'Mascot / inflatable / animal / character / robot-armor / oversized / large-prop costumes break silhouette, texture and proportion priors, causing misses or misclassification (vehicle, animal, background). The Rare-Event Miner keeps three SEPARATE confidences: a human is present, a costume is present, and the combination is a rare event.', caveat: 'Confounders (mascot statues, inflatable decorations, printed figures, mannequins) look identical in one frame — only motion, 3D volume, or human validation separates them.', }, leakage_guard: { term: 'Leakage guard', category: 'Operations', short: 'Examples in protected evaluation sets can never silently become training data.', detail: 'Anything routed to a REGRESSION or SAFETY_CRITICAL evaluation set gets training_eligible=false in its lineage. Promoting such an example to training requires an explicit governance override recording who and why, which is permanently audited.', }, mining_calibration: { term: 'Mining calibration', category: 'Anomaly & rarity', short: 'Whether the miner\u2019s stated confidence matches how often it is right.', detail: 'Candidates are binned by stated rare-event confidence; within each bin, the observed rate of true (planted) rare events is measured. A calibrated miner\u2019s 80% bin is right about 80% of the time — over-confident bins mean stated confidence cannot be trusted for triage ordering.', }, // ------------------------------------------------------------ EM readiness (Hill Climbing EM) hc_competency: { term: 'Competency', category: 'EM readiness', short: 'A single skill node in the 4-phase blueprint graph, with prerequisites and a dimension tag.', detail: 'Each competency belongs to a phase (ML depth / system design / execution & people / simulation), lists prerequisite competencies, and is tagged with one dimension (Knowledge, Technical Reasoning, Leadership, Execution). Scores are tracked per competency and per dimension — never collapsed into one number.', }, hc_readiness_state: { term: 'Readiness state', category: 'EM readiness', short: 'Per-competency progression: NOT_STARTED → LEARNING → PRACTICING → NEEDS_REVIEW → COMPETENT → STRONG → INTERVIEW_READY.', detail: 'Derived from three separate scores: knowledge (diagnostic/interview answers), application (exercises, design lab, simulation) and evidence (STAR stories and other artifacts). NEEDS_REVIEW is entered when recent attempts contradict earlier strong scores.', caveat: 'States are only as trustworthy as the evidence backing them; with the LLM offline, scoring is rule-based concept coverage, which is stricter but coarser.', }, hc_evidence_artifact: { term: 'Evidence artifact', category: 'EM readiness', short: 'A stored, quotable record (attempt, STAR story, design grade, simulation debrief, interview transcript) that justifies a score.', detail: 'The scoring rule is "no score without evidence": every evaluation must quote specific user statements. Artifacts persist under runs/hillclimb/ and are browsable in the Evidence Library; clicking any matrix score shows the artifacts behind it.', }, hc_bottleneck: { term: 'Bottleneck competency', category: 'EM readiness', short: 'The weak prerequisite blocking the most downstream competencies — not merely the lowest score.', detail: 'Computed from the prerequisite graph: for each weak competency, count the downstream competencies transitively gated on it; the highest-leverage weakness wins. Fixing the bottleneck unblocks more of the graph than fixing the globally lowest score.', }, hc_next_best_action: { term: 'Next best action', category: 'EM readiness', short: 'Exactly one concept to study, one exercise to attempt and one assessment to take, aimed at the current bottleneck.', detail: 'Regenerated whenever the readiness matrix changes. Deliberately singular: a ranked to-do list invites cherry-picking easy items; one action per category forces work on the highest-leverage weakness.', }, hc_claim_vs_evidence: { term: 'Claim vs evidence', category: 'EM readiness', short: 'STAR Story Box flag: an unquantified claim ("improved performance") vs measurable evidence (numbers, before/after, named mechanism).', detail: 'Claim sentences are detected by improvement verbs without attached quantities. Each flag asks for the missing strengthening: the metric, the baseline, the delta, or the mechanism that caused it. Interviewers apply the same test — so the diagnoser applies it first.', }, hc_anti_gaming: { term: 'Anti-gaming rule', category: 'EM readiness', short: 'Verbosity alone must not raise scores: concept coverage, tradeoffs and quantified results are scored, not length.', detail: 'The evaluator scores rubric-concept coverage, tradeoff discussion and quantified statements. Length is not a feature, and low information density is penalized — a long waffle scores below a short precise answer (enforced by test).', }, hc_hill_climbing: { term: 'Hill climbing (simulation)', category: 'EM readiness', short: 'Iterative optimization loop: hypothesis → intervention → measure → keep/reject → repeat, under competing objectives.', detail: 'The Phase-4 simulation tracks ten competing state metrics (performance, safety, reliability, cost, velocity, maintainability, morale, customer impact, risk, schedule) with hard floors — dropping safety below its floor triggers an incident. The balanced multi-objective score rewards keeping all objectives healthy, not maximizing one.', caveat: 'Deterministic given the scenario seed: identical intervention sequences replay identically, which is what makes debriefs auditable.', }, // ------------------------------------------------------------ hardware acceleration (Vitis) vitis_emulated_backend: { term: 'Emulated Vitis backend', category: 'Hardware acceleration', short: 'CPU emulator of a Vitis Vision FPGA pipeline: same ops, faithful hardware constraints, no silicon.', detail: 'Implements the same VisionBackend interface as the reference float32 backend, but applies ap_fixed fixed-point quantization (truncation + saturation), XFCVDEPTH line-buffer limits and LUT divide/sqrt to every op, plus a deterministic per-op latency model (pixels/cycle × clock, PL vs AIE placement). A real Vitis/PYNQ/XRT backend slots in behind the identical interface later.', caveat: 'No FPGA hardware is attached to this machine. Every latency, throughput and speedup figure this backend reports is analytically MODELED, never measured on silicon.', }, ap_fixed_quantization: { term: 'ap_fixed quantization', category: 'Hardware acceleration', short: 'HLS fixed-point format: W total bits, I integer bits; values are truncated and saturated like on the FPGA.', detail: 'Matches the HLS ap_fixed defaults: rounding is truncation toward negative infinity (AP_TRN) at 2^-(W-I) resolution, and overflow saturates (AP_SAT) to the representable range. Fewer fractional bits mean coarser detection confidences and positions — the root cause the HIL ablation isolates as "precision".', }, xfcvdepth: { term: 'XFCVDEPTH (line-buffer depth)', category: 'Hardware acceleration', short: 'Maximum image width a streaming Vitis Vision kernel can buffer rows for.', detail: 'Streaming FPGA kernels hold a few rows in on-chip line buffers whose width is bounded at synthesis time. When the frame is wider than the configured depth, the emulator processes independent vertical strips with no halo exchange — producing localized seam artifacts at strip boundaries, exactly like an under-provisioned kernel.', }, lut_approximation: { term: 'LUT divide/sqrt', category: 'Hardware acceleration', short: 'HLS-style lookup-table approximation of divide and square root with bounded relative error.', detail: 'Full dividers are expensive in fabric, so HLS designs use mantissa-indexed lookup tables. The emulator uses 2^lut_bits-entry tables (relative error ≈ 2^-lut_bits) for every reciprocal and square root — including the determinant inverse inside the optical-flow solver.', }, quantization_gap_score: { term: 'Quantization gap score', category: 'Hardware acceleration', short: 'Scalar severity of the float32-vs-fixed-point detection gap on identical frames.', detail: 'Computed from paired per-object comparisons: 0.45·dropped-detection rate + 0.2·class-flip rate + 0.2·mean |confidence drift| + 0.15·(1 − mean pair IoU). The HIL ablation re-measures it with one hardware constraint enabled at a time to attribute the gap to precision, streaming depth, or LUT approximation.', }, modeled_throughput: { term: 'Modeled FPGA throughput', category: 'Hardware acceleration', short: 'Analytical latency estimate (pixels/cycle × clock MHz), clearly labeled — never a measurement.', detail: 'Each op has a modeled pixels/cycle rate and PL-vs-AIE placement; end-to-end pipelined fps assumes HLS dataflow overlap (1 / max stage latency). Reported alongside the MEASURED CPU wall time of the reference pipeline for contrast.', caveat: 'Ignores memory bandwidth, AXI backpressure and resource contention; it is an upper bound to be validated on silicon, and every report carries modeled_not_measured: true.', }, flow_motion_baseline: { term: 'Flow motion baseline', category: 'Hardware acceleration', short: 'Model-independent object motion predicted by dense pyramidal Lucas-Kanade optical flow.', detail: 'Flow over consecutive BEV rasters predicts each ground-truth object\u2019s next position without consulting any detection engine. A track is "flow-continuous" at a frame when the flow-predicted displacement matches the true displacement within a residual gate — the license to blame the engine (not the scene) for a dropped detection.', }, temporal_stability_score: { term: 'Temporal stability score', category: 'Hardware acceleration', short: 'Per-engine 0–100 score penalizing flicker, jitter, fragmentation and unexcused ID switches against the flow baseline.', detail: '100 × (0.35·(1 − 4·flicker rate) + 0.25·e^(−1.5·jitter) + 0.25·(1 − 0.6·fragmentation/track) + 0.15·(1 − unexcused ID-switch fraction)). Computed per cohort (day/night/rain/occluded) and per backend; the backend-agreement meta-check verifies fixed-point flow does not change the engine ranking.', caveat: 'The weights are a documented product decision; check ranking sensitivity before gating releases on the absolute score.', }, evaluation_only_variant: { term: 'Evaluation-only variant', category: 'Hardware acceleration', short: 'Synthetic stress variant barred from training by default: full lineage, protected eval destination.', detail: 'Every generated augmentation variant carries training_eligible: false, evaluation_only: true and destination REGRESSION_EVALUATION_SET, plus complete lineage (recipe with resolved parameters, seed, source frame, backend config). Mirrors the rare-event miner\u2019s protected-destination leakage guard: stress data that leaked into training would corrupt the very regression signal it exists to protect.', }, // ------------------------------------------------------------ safety & compliance odd: { term: 'ODD (Operational Design Domain)', category: 'Safety & compliance', short: 'The set of conditions the autonomy system is designed — and validated — to operate in.', detail: 'Formalized here as a taxonomy of dimensions (object class, weather, lighting, traffic density, road type). Every claim of safety only holds inside the ODD cells that have adequate evaluation evidence.', }, odd_coverage: { term: 'ODD coverage', category: 'Safety & compliance', short: 'Fraction of ODD cells with enough evaluation samples and acceptable performance.', detail: 'A cell (class × condition) counts as covered when its evaluation sample count meets the per-cell requirement and its recall clears the policy floor. Production-weighted coverage weights each cell by its real-world exposure frequency, so covering a common cell counts more than an exotic one.', caveat: 'Coverage measures evidence quantity, not difficulty: a covered cell can still hide a hard sub-population.', }, coverage_gap: { term: 'Coverage gap', category: 'Safety & compliance', short: 'An ODD cell with too little evaluation evidence (or too-low performance) to support a safety claim.', detail: 'Gaps are ranked by risk score = production exposure × class severity × statistical uncertainty. The top-ranked gap is the highest-value target for scenario generation or data collection.', }, release_gate: { term: 'Release gate', category: 'Safety & compliance', short: 'A named safety criterion that must pass before a model can be promoted.', detail: 'Each gate bundles concrete checks (measured value vs policy threshold): overall quality, safety-critical recall, non-regression vs baseline, ODD coverage, and calibration health. One failing check blocks the gate; one blocked gate blocks the release.', }, evidence_package: { term: 'Safety evidence package', category: 'Safety & compliance', short: 'A structured document assembling all evaluation evidence behind a release decision.', detail: 'Generated per candidate run: scope, gate results, coverage summary, calibration status, known limitations and sign-off placeholders, in the spirit of ISO 21448 (SOTIF) / UL 4600 argumentation.', caveat: 'The package generated in this demo is a synthetic demonstration artifact — not a real certification document.', }, drac: { term: 'DRAC', category: 'Safety (SSAM)', short: 'Deceleration Rate to Avoid Crash — braking intensity required to prevent the collision.', detail: 'Computed from relative speed and gap at the conflict point: DRAC = Δv² / (2·distance). Values above ~3.35 m/s² conventionally mark an emergency-braking-level conflict.', }, delta_s: { term: 'ΔS (speed differential)', category: 'Safety (SSAM)', short: 'Relative speed of the two road users at the conflict point.', detail: 'Higher ΔS means higher kinetic energy in a hypothetical collision — it scales the severity component of conflict indices.', }, csi: { term: 'CSI (Conflict Severity Index)', category: 'Safety (SSAM)', short: 'Composite 0–1 severity score combining proximity (TTC/PET), required braking (DRAC) and speed differential.', detail: 'Blends the surrogate measures into a single ranking score so conflicts from different mechanisms (crossing, rear-end, merging) are comparable. Used to rank conflicts and compare scenarios.', caveat: 'The blend weights are a documented convention; always inspect the underlying TTC/PET/DRAC values before acting on a CSI ranking.', }, extrinsic_calibration: { term: 'Extrinsic calibration', category: 'Safety & compliance', short: 'The rigid transform (rotation + translation) mapping one sensor\u2019s frame into another\u2019s.', detail: 'Validated cross-sensor: project LiDAR objects into the camera frame and measure residual offsets. A consistent directional residual across all objects indicates miscalibration; large residuals on a few objects indicate perception failures.', }, residual: { term: 'Calibration residual', category: 'Safety & compliance', short: 'Per-object positional disagreement between camera and LiDAR after applying the calibration transform.', detail: 'Plotted as a 2D scatter (lateral × longitudinal offset). Healthy: tight cloud at origin. Miscalibrated: whole cloud shifted/rotated coherently. Perception failure: tight cloud plus isolated far outliers.', }, discrepancy_mining: { term: 'Discrepancy mining', category: 'Safety & compliance', short: 'Systematically finding objects where auto-labels, human labels and model predictions disagree.', detail: 'Disagreements are typed (missing box, spurious box, class confusion, geometry drift) and aggregated per cohort. A cohort with a high discrepancy rate is where labeling quality is systematically weakest — without needing any new ground truth.', }, semantic_search_stage: { term: 'Hybrid semantic search', category: 'Safety & compliance', short: 'Concept search = symbolic dimension filter → reasoning scorer → blended ranking, with per-stage explanations.', detail: 'The symbolic stage parses hard constraints from the query (e.g. lighting=night); the reasoning stage scores conceptual relevance; the blend combines them. Each result explains its score per stage, and a provider badge states whether an LLM or the deterministic fallback did the scoring.', }, // ------------------------------------------------------------ sequential testing evalue: { term: 'E-value / e-process', category: 'Sequential testing', short: 'Betting-style evidence measure that stays valid under continuous monitoring — no peeking penalty.', detail: 'An e-process accumulates multiplicative evidence against a hypothesis; its running value can be checked after every sample, and rejecting when it crosses 1/α controls the error rate at α regardless of when you stop. The sequential-evidence chart plots log e-values, so the stopping boundary is a horizontal line.', }, stopping_boundary: { term: 'Stopping boundary', category: 'Sequential testing', short: 'The evidence threshold that ends the sequential test the moment it is crossed.', detail: 'Crossing the upper (rejection) boundary = enough evidence of regression → BLOCK; crossing the lower boundary = enough evidence of safety → PASS. Running out of budget between the boundaries yields INSUFFICIENT_EVIDENCE.', }, n_eff: { term: 'Effective n', category: 'Sequential testing', short: 'The information-equivalent sample count after accounting for correlation and weighting.', detail: 'Samples that are correlated (same scene, same track) carry less independent information than their raw count suggests. Decisions report both raw n used and effective n; a large gap flags heavy within-cluster correlation.', }, safety_primary_node: { term: 'Safety-primary node', category: 'Sequential testing', short: 'A tested slice flagged as safety-critical and held to stricter sequential thresholds.', detail: 'Safety-primary nodes (e.g. pedestrian-at-night) get tighter regression tolerances and higher evidence requirements — a marginal regression that would pass on a comfort slice blocks on a safety-primary one.', }, sample_budget: { term: 'Sample budget', category: 'Sequential testing', short: 'The maximum labeled samples the sequential run may spend before it must decide.', detail: 'The budget funnel shows planned budget vs actually used vs the full population size: the gap between used and population is the labeling cost the sequential design saved over a fixed-n evaluation.', }, // ------------------------------------------------------------ perception fusion bev: { term: 'BEV (bird\u2019s-eye view)', category: 'Perception fusion', short: 'Top-down ego-centric view of the scene — the natural frame for fusing multi-sensor detections.', detail: 'All detections and boxes are expressed in the ego frame (x forward, y left, meters) and rasterized or drawn top-down. Distance rings and an ego marker anchor the geometry.', }, along_ray_uncertainty: { term: 'Along-ray uncertainty', category: 'Perception fusion', short: 'Monocular camera depth error: large along the viewing ray, small across it.', detail: 'A camera fixes bearing precisely but estimates depth poorly, so its position covariance is an elongated ellipse pointing along the ray from the camera — exactly what the BEV canvas draws. Fusion with LiDAR (precise range) collapses that ellipse.', }, masklet_propagation: { term: 'Masklet propagation', category: 'Perception fusion', short: 'Carrying a track through detection gaps using its motion model instead of dropping it.', detail: 'When a tracked object gets no detection this frame (occlusion, dropout), the tracker propagates the box along the track\u2019s estimated velocity for a bounded number of frames — drawn dashed on the canvas. Identity survives the gap, which is what improves IDF1 and cuts fragmentation.', caveat: 'Propagation is a prediction, not an observation: propagated boxes carry reduced confidence and expire if no detection re-associates.', }, fused_track: { term: 'Fused track', category: 'Perception fusion', short: 'An object hypothesis maintained over time from camera + LiDAR detections fused in BEV.', detail: 'Camera and LiDAR detections are associated and merged per frame (covariance-weighted), then linked over time by the tracker. The canvas shows fused boxes filled, with the track ID label — click one to follow it across frames.', }, // ------------------------------------------------------------ retrospective safety evidence_tier: { term: 'Evidence tier', category: 'Retrospective safety', short: 'Every claim in a retrospective is tagged by how it was produced: observed, derived, retrieved, or hypothesized.', detail: 'Tier 1 OBSERVED = a fact read directly from the failure log. Tier 2 DERIVED = deterministically computed from observed facts (stopping distance, TTC, SCR impact). Tier 3 RETRIEVED = text returned by the safety-case RAG with full citation. Tier 4 AI_HYPOTHESIS = an LLM inference, never treated as fact. The scorecard renders the distinction explicitly.', caveat: 'A hypothesis with high confidence is still Tier 4 — confidence never promotes an inference to a fact.', }, retro_severity: { term: 'Retrospective severity', category: 'Retrospective safety', short: 'BENIGN / DISRUPTIVE / CRITICAL / FATAL — computed deterministically from contextual evidence, not by the LLM.', detail: 'An asymmetric cost model: false negatives are scored by VRU class × safety-relevant distance × relative motion × remaining reaction time; false positives by intervention magnitude × traffic disruption. The LLM may propose a severity, but the deterministic policy engine validates or overrides it, and any divergence is flagged for human review.', caveat: 'Not every miss outranks every phantom: a distant benign FN ranks below a hard-braking phantom FP with tailgating traffic.', }, launch_gate_retro: { term: 'Launch recommendation gate', category: 'Retrospective safety', short: 'Deterministic, policy-versioned mapping from severity + evidence completeness to PASS / CONDITIONAL_PASS / FAIL / INSUFFICIENT_EVIDENCE.', detail: 'The gate is pure code with a versioned config: FATAL/CRITICAL fail, DISRUPTIVE passes conditionally, and missing critical telemetry yields INSUFFICIENT_EVIDENCE — which can never be upgraded to PASS. The recommendation is advice to a human decision-maker, never an automatic launch.', }, synthetic_standard: { term: 'SYNTHETIC_EXAMPLE standard', category: 'Retrospective safety', short: 'Every document in the safety-case corpus is a labeled synthetic demonstration — never real standard text.', detail: 'Corpus documents (safety requirements, launch criteria, ODD definitions, historical retrospectives) carry SYNTHETIC_EXAMPLE / NOT_A_REAL_STANDARD in both content and metadata; SOTIF entries are concept-level paraphrases explicitly labeled as not standard text. The retriever surfaces these labels with every citation.', caveat: 'Hard rule: the system never cites a safety requirement without a retrieval hit backing it.', }, tool_audit_trail: { term: 'Tool audit trail', category: 'Retrospective safety', short: 'Every agent tool call — including denials — is appended to a per-analysis audit log with args, result hash, and timestamp.', detail: 'Tools are read-only by default; the single write tool (CreateEvaluationCase) requires an explicit policy authorization flag. The log reader is path-allowlisted to runs/ and the fixtures directory. The audit trail makes each scorecard reproducible and reviewable.', }, data_label: { term: 'Data provenance label', category: 'Closed-loop evaluation', short: 'Every evaluation datum is labeled REAL / REPLAYED / SIMULATED / GENERATED / COUNTERFACTUAL — carried through all reports.', detail: 'Simulation is evidence, not reality. The label states exactly how a scenario came to exist: REAL sensor data, REPLAYED logs, SIMULATED synthetic scenes, GENERATED novel scenarios, or COUNTERFACTUAL transformations of an existing scene. Launch recommendations list the labels of every stratum that fed them.', caveat: 'A COUNTERFACTUAL result never carries the evidentiary weight of a REAL one — the validity gate quantifies (fidelity, realism) but cannot erase the gap.', }, counterfactual_scenario: { term: 'Counterfactual scenario', category: 'Closed-loop evaluation', short: 'A recorded scene deterministically transformed ("what if it were foggy / the lead car braked") with full provenance: source scene, recipe, seed.', detail: 'Transformations cover environment (fog, rain, night, glare), actor behavior (sudden brake, swerve, crossing, occluded emergence), and scene structure (construction zone, pedestrian density, occlusion). Same recipe + seed reproduces the identical scenario.', caveat: 'Only gate-accepted counterfactuals enter evaluation suites, and low-fidelity ones are weight-capped.', }, validity_gate: { term: 'Counterfactual validity gate', category: 'Closed-loop evaluation', short: 'Five deterministic checks — physical plausibility, temporal consistency, sensor consistency, identity/trajectory consistency, distribution similarity — before a generated scenario may enter a suite.', detail: 'Produces simulation_fidelity_score, counterfactual_validity, and realism_confidence. Realism measures the shift ADDED by the transformation relative to the source scene (PSI/JS via the mega-eval statistics), so a valid transformation of an unusual scene is not unfairly penalized.', }, scr: { term: 'Safety-Critical Recall (SCR)', category: 'Closed-loop evaluation', short: 'Recall restricted to objects inside the parameterized safety-critical region — where a miss can actually change the outcome.', detail: 'The region is NOT naive stopping distance: d_crit(v) = v·t_react + v²/(2·min(brake, μg)) + buffer, with a lateral band scaled by per-class encroachment speed and a projected-TTC criterion for crossing actors. All parameters (reaction time, brake capability, friction) are explicit and reported with every result.', caveat: 'Overall recall can improve while SCR degrades — the divergence demo on the Closed-Loop Lab page constructs exactly that case.', }, causal_replay: { term: 'Causal counterfactual replay', category: 'Closed-loop evaluation', short: 'Run the same scenario twice — actual perception vs GT-corrected — and diff behavior to classify a regression as METRIC_ONLY or BEHAVIORALLY_CONSEQUENTIAL.', detail: 'The causal chain is answered stepwise: would correct perception change the planner output? would that change vehicle behavior? would that change the safety outcome? Only a yes on the final step makes an error behaviorally consequential.', caveat: 'Verdicts are relative to the simplified deterministic planner/controller; a production planner may react differently. The verdict is evidence for triage, not proof.', }, behavioral_metrics: { term: 'Closed-loop behavioral metrics', category: 'Closed-loop evaluation', short: 'Detection latency, TTC, stopping distance, deceleration, planner interventions, minimum separation, safety margin — measured by running the full perception→planner→vehicle loop.', detail: 'Complementary to open-loop metrics, never a replacement: every closed-loop report also carries frame recall. Safety math (TTC, stopping distance) reuses the SSAM safety extensions.', }, anytime_valid_stopping: { term: 'Anytime-valid early stopping', category: 'Closed-loop evaluation', short: 'The gauntlet stops a stratum early only on evidence measures that remain valid under continuous monitoring (e-process / confidence sequences via seqeval).', detail: 'Naive repeated significance testing inflates false alarms; the scheduler delegates every stop/continue decision to seqeval sequential tests, so a catastrophic-regression halt after 2k of 12k units carries the same statistical guarantee as a full run.', }, compute_dedup: { term: 'Compute deduplication', category: 'Closed-loop evaluation', short: 'Content-addressed caching of shared pipeline stages (BEV rasters, projections, backbone features) so N models re-run only their heads.', detail: 'Cache keys hash (scenario, sensor_version, preprocessing_version, backbone_version, feature_schema_version); any version bump changes the key, so stale cross-version reuse is structurally impossible. Savings are measured, not estimated.', }, // ------------------------------------------------------------ launch readiness (agentic triage) policy_outcome: { term: 'Policy outcome (four-way)', category: 'Launch readiness', short: 'AUTOMATIC_STOP_SHIP / LAUNCH_REVIEW_REQUIRED / CONTINUE_INVESTIGATION / NO_LAUNCH_IMPACT — decided by the deterministic policy engine, never by an agent.', detail: 'The versioned stop-ship policy evaluates explicit conditions over severity × exposure × frequency × confidence × novelty × downstream consequence. Automatic stop-ship fires only on pre-authorized conditions; anything with missing ground truth, incomplete lineage, failed fusion verification or conflicting agents resolves to INDETERMINATE (a fail-safe escalation), never a pass.', caveat: 'All numeric thresholds in the shipped policy are example placeholders (flagged placeholder_values: true); a real deployment must set organization-approved values through the versioned policy API.', }, severity_taxonomy: { term: 'Severity taxonomy (S0–S5)', category: 'Launch readiness', short: 'S0 cosmetic → S5 collision-in-replay, assigned by deterministic criteria over class criticality, containment and observed behavioral impact.', detail: 'S4 requires OBSERVED unsafe downstream behavior in replay evidence — a hypothesized consequence cannot reach S4. S3 is a safety-critical misclassification whose downstream effect is not shown to be contained.', }, evidence_status_tag: { term: 'Evidence status (OBSERVED / DERIVED / HYPOTHESIS / UNAVAILABLE)', category: 'Launch readiness', short: 'Every node in the failure evidence graph carries an explicit epistemic status; missing evidence is shown as UNAVAILABLE rather than guessed.', detail: 'OBSERVED = measured from campaign data or replay; DERIVED = computed deterministically from observed data; HYPOTHESIS = produced by an advisory agent and requires confirmation; UNAVAILABLE = the evidence does not exist yet. Agents cannot upgrade their own hypotheses.', }, option_c_reduced_odd: { term: 'Option C — reduced ODD', category: 'Launch readiness', short: 'Ship while excluding the concentrated failure region from the operational design domain — allowed only when the region is reliably detectable at runtime.', detail: 'Feasibility checks use the ODD taxonomy: the concentrating dimension must be an instrumented ODD dimension, the runtime detector recall must meet MIN_REQUIRED_SAFETY_RECALL, the post-exclusion residual rate must meet MAX_ACCEPTABLE_FAILURE_PROBABILITY, evidence confidence must reach the configured minimum, and the exclusion must not exceed MAX_ALLOWED_ODD_EXCLUSION.', caveat: 'A geographic bucket is not an instrumented ODD dimension, so geo-concentration alone can never justify Option C.', }, hard_safety_constraint: { term: 'Safety as a hard constraint', category: 'Launch readiness', short: 'Expected loss is computed for every option, but options whose residual failure rate violates the policy limit are excluded from selection no matter how cheap they are.', detail: 'Business optimization happens only inside the safe feasible region: the option matrix skips a matched row whose option is safety-infeasible, and if no feasible row remains the engine fails safe to human safety review. Tested explicitly: a cheaper unsafe option is never selected.', }, contamination_guard: { term: 'Contamination guard', category: 'Launch readiness', short: 'Members of flywheel evaluation suites are never training-eligible without an explicit recorded override (who + why).', detail: 'Prevents the learning flywheel from leaking its own evaluation examples into training data, which would silently invalidate future evaluations. Promotion without an actor and reason raises a leakage error (reusing the raremine lineage guard); overrides are recorded on the suite and in the audit trail.', }, small_sample_instability: { term: 'Small-sample instability flag', category: 'Launch readiness', short: 'Raised when event counts are too small for the rate estimate to be stable (e.g. 25 events on a 240k denominator), capping evidence confidence.', detail: 'A ~0.01% rate estimated from a handful of events has a wide Wilson interval and an unstable relative risk. The statistical agent flags this, the evidence-quality level drops to LIKELY or below, and automatic stop-ship (which requires CONFIRMED) cannot fire on it — the case routes to human review instead.', }, audit_hash_chain: { term: 'Hash-chained audit log', category: 'Launch readiness', short: 'Every stage transition, agent output, policy evaluation and human decision is appended to a per-failure JSONL log where each record hashes its predecessor.', detail: 'Records carry a monotonically increasing sequence number and a prev_hash → hash chain; any edit, deletion or reordering breaks recomputation and is reported as a broken chain. The log is append-only by construction.', }, // ------------------------------------------------------------ release governance (studio2) release_decision: { term: 'Release decision (GO / REVIEW / NO-GO)', category: 'Release governance', short: 'One deterministic decision composed from safety gates, the seqeval regression verdict, the megaeval shift report, and (when available) agentic policy and closed-loop verdicts.', detail: 'Computed by the studio2 ReleaseGate as a pure function of the subsystem inputs plus a content-hash-versioned policy. Any blocking condition (safety BLOCKED, seqeval REGRESSION, agentic stop-ship, failing critical hardware combination) → NO-GO; missing inputs or open questions → REVIEW with the gap named; only complete, clean evidence → GO. Every decision records the full evidence tuple it was computed from.', caveat: 'GO is a recommendation, never a deployment: authorization is a separate recorded human approval.', }, evidence_completeness: { term: 'Evidence completeness', category: 'Release governance', short: 'The fraction of configured release-gate subsystems that actually contributed evidence to a decision.', detail: 'Required subsystems (safety gates, sequential regression, distribution shift) and configured optional ones (agentic policy, closed loop, hardware matrix) each count once. A missing subsystem lowers completeness and forces REVIEW — absence of evidence can never make a decision look stronger.', }, reproducibility_tuple: { term: 'Reproducibility tuple', category: 'Release governance', short: 'The seven components needed to replay an evaluation run exactly: model, dataset, scenario, config, calibration, seed and policy versions.', detail: 'Recorded on every EvaluationRun in the studio2 registry. The verdict is computed, not declared: a run missing any component is marked NON_REPRODUCIBLE with the missing parts listed (most retroactively ingested runs are, honestly, non-reproducible — e.g. megaeval lineage lacks scenario and calibration components).', }, dataset_role: { term: 'Dataset role', category: 'Release governance', short: 'Every dataset version carries a role: TRAINING, VALIDATION, TEST, REGRESSION, LAUNCH or MONITORING.', detail: 'TEST, REGRESSION and LAUNCH are protected evaluation roles: they decide launches, so moving them across the training/evaluation boundary (in either direction) requires an explicitly recorded governance override (who + why), enforced by the registry\u2019s role-transition rules — the same contamination-guard pattern as the raremine lineage LeakageError.', }, hardware_gate_matrix: { term: 'Hardware gate matrix', category: 'Release governance', short: 'Metric thresholds evaluated per hardware/domain combination (region × compute platform × sensor generation), not just globally.', detail: 'Combination metrics come from real megaeval cube cohorts; platform evidence exists only where vitis HIL runs exist; region strata are proxied from road_type (the derivation is shown per row). A failing launch-critical combination blocks the release even when the global aggregate passes; combinations without enough support are INSUFFICIENT and never count as passing.', }, human_deployment_approval: { term: 'Human deployment approval', category: 'Release governance', short: 'The separate recorded action (approver + rationale) that authorizes deployment after a GO decision.', detail: 'Only GO decisions can be approved; REVIEW and NO-GO must be re-evaluated after their conditions are resolved. The approval is a first-class registry entity and lands in the append-only audit trail — the gate recommends, humans authorize.', }, hardening_finding: { term: 'Hardening finding', category: 'Production hardening', short: 'One audited defect or gap with FILE:LINE references, a severity (Critical/High/Medium/Low), and a disposition (fixed now vs documented follow-up).', detail: 'Findings live in docs/hardening/audit.json and drive everything on the Production Readiness page. "Fixed now" items were implemented surgically (API-preserving) or as a layered capability in sensorflow/hardening; follow-ups carry an explicit reason (usually an API-breaking change or the concurrent-edit freeze on app_backend.py).', }, readiness_scorecard: { term: 'Readiness scorecard', category: 'Production hardening', short: 'Category-level prototype-vs-production gap grid, computed from the audit findings — never hand-written.', detail: 'Each category maps its findings to a status (blocked-critical / gaps-open / partially-hardened / closed). The overall verdict is hard-wired to NOT PRODUCTION READY while any Critical finding is open, so the status cannot be flipped without closing findings.', }, label_provenance: { term: 'Label provenance', category: 'Production hardening', short: 'Where a label came from: MODEL_PREDICTION → AUTO_LABEL / VLM_INFERENCE → HUMAN_LABEL → CERTIFIED_GROUND_TRUTH, in increasing order of trust.', detail: 'Defined in sensorflow/hardening/contracts.py and mandatory on every hardening contract. Nothing lower on the ladder may silently substitute for a level above it — e.g. a quality gate must refuse AUTO_LABEL "ground truth". Adapters map existing records (labeleval GT types, pipeline proposals) onto the ladder.', caveat: 'Existing packages are not yet rewritten to these contracts; adapters bridge until their next breaking release.', }, rotr_violation: { term: 'ROTR violation', category: 'ROTR (right-of-the-road)', short: 'A deterministic rule-engine verdict that the executed trajectory broke the expected-behavior envelope (yielding, legal path, right-of-way, stopping).', detail: 'Computed as a function of five evidence structures: ego state, road context, actor states, observed behavior and expected behavior. Each record carries a versioned rule id, structured evidence fields and provenance. Detection runs against the ACTUAL (ground-truth) world; plan compliance runs separately against the stack\u2019s believed world.', caveat: 'A violation verdict says nothing about cause or consequence — those are separate analyses (attribution, counterfactual replay).', }, rotr_attribution: { term: 'Causal-layer attribution', category: 'ROTR (right-of-the-road)', short: 'Per-violation, per-layer evidence tests: each of perception / prediction / planning / localization / map / control / data-label / policy-rule is independently SUPPORTED, RULED_OUT or UNKNOWN.', detail: 'The load-bearing invariant: a violation is NEVER auto-attributed to perception. Each layer needs its own positive evidence (e.g. GT-vs-detected diff for perception, correct-perception-but-noncompliant-plan for planning). The primary layer is the highest-confidence SUPPORTED entry; with none, the row stays unattributed and goes to HITL.', }, rotr_consequence: { term: 'Behavioral consequence class', category: 'ROTR (right-of-the-road)', short: 'What the violation actually cost, measured by counterfactual replay: NO_MATERIAL_CONSEQUENCE, DEGRADED_COMFORT, PLANNER_INTERVENTION or SAFETY_CRITICAL.', detail: 'The attributed layer\u2019s inputs are corrected and the scenario is re-planned (nextgen closed-loop when importable, else a self-contained IDM-style planner). Observed vs corrected trajectories are compared on TTC, PET, clearance, stopping distance, braking intensity and lateral deviation.', caveat: 'Surrogate-safety measures (TTC/PET/clearance) indicate exposure, not collision certainty — they alone do not establish collision risk.', }, sc_rotr_recall: { term: 'SC-ROTR recall', category: 'ROTR (right-of-the-road)', short: 'Safety-critical-weighted violation recall: misses on dangerous violations cost more than misses on cosmetic ones.', detail: 'Weights are exposure-derived per stratum — the fraction of that stratum\u2019s violations whose counterfactual consequence was safety-critical, normalized so weights average 1 — with the calibration method and version recorded next to the number. Not arbitrary class multipliers.', caveat: 'On synthetic banks the weights are calibrated on the bank itself; a production system would calibrate on fleet exposure data.', }, rotr_stopship: { term: 'ROTR stop-ship trigger', category: 'ROTR (right-of-the-road)', short: 'Deterministic NO_GO on the strict conjunction: VRU involved AND perception missed the actor AND counterfactual consequence is safety-critical.', detail: 'Policy-versioned and explicitly not LLM-driven. Partial matches never fire. NO_GO events are forwarded to the agentic policy engine as advisory input when importable; the gate outcome is recorded locally either way.', }, rotr_six_axis: { term: 'Six-axis ROTR taxonomy', category: 'ROTR (right-of-the-road)', short: 'Actor / Vulnerability / Legality / Environment / Interaction / Behavior — plus road-geometry, traffic-control and visibility extensions — for structured violation mining.', detail: 'Text like "failed to yield to pedestrian at uncontrolled intersection during low visibility" parses through a deterministic keyword map into the same filterable query object as the dropdown chips. Violations cluster into recurring patterns by structured signature.', }, }; export type GlossaryKey = keyof typeof GLOSSARY; /** Resolve a status / reason string (e.g. "FLAGGED", "LOW_IOU") to a glossary key, if known. */ export function glossaryKeyForStatus(status: string): string | null { const s = status.toLowerCase(); const direct: Record = { auto_graded: 'status_auto_graded', flagged: 'status_flagged', verified: 'status_verified', rejected: 'status_rejected', pending: 'status_pending', low_iou: 'reason_low_iou', position_error: 'reason_position_error', orientation_error: 'reason_orientation_error', insufficient_point_support: 'reason_insufficient_point_support', sensor_disagreement: 'reason_sensor_disagreement', anomaly: 'reason_anomaly', grader_disagreement: 'reason_grader_disagreement', id_switch: 'reason_id_switch', track_fragmentation: 'reason_track_fragmentation', low_confidence: 'reason_low_confidence', model_regression: 'reason_model_regression', tp: 'error_tp', fn: 'error_fn', fp: 'error_fp', localization: 'error_localization', low_conf: 'error_low_conf', pseudo_ground_truth: 'gt_pseudo', vendor_ground_truth: 'gt_vendor', human_verified_ground_truth: 'gt_human_verified', gold_standard: 'gt_gold', }; return direct[s] ?? null; } /** Glossary keys for gate ids as they appear in gate lines. */ export const GATE_GLOSSARY: Record = { iou_3d: 'iou_3d', position_error_m: 'position_error', orientation_error_deg: 'orientation_error', has_reference_gt: 'gt_coverage', points_in_box: 'points_in_box', box_occupancy: 'point_in_box_ratio', centroid_consistency_m: 'position_error', ground_contact_error_m: 'ground_contact', box_dimensions_vs_class_prior: 'dimension_error', dimension_error: 'dimension_error', sensor_consistency_cam_lidar: 'sensor_consistency', sensor_consistency: 'sensor_consistency', geometric_validation: 'quality_gate', anomaly_score: 'anomaly_score', grader_consensus: 'grader_consensus', id_switch: 'id_switch', track_fragmentation: 'fragmentation', track_quality: 'track_quality', detection_confidence: 'confidence', model_regression: 'reason_model_regression', }; export const GLOSSARY_CATEGORIES: GlossaryCategory[] = [ 'Detection metrics', 'Geometry & LiDAR', 'Tracking', 'Anomaly & rarity', 'Safety (SSAM)', 'Human grading & sampling', 'Aggregation & scale', 'Pipeline & statuses', 'Failure reasons', 'Ground truth', 'Operations', 'EM readiness', 'Hardware acceleration', 'Safety & compliance', 'Sequential testing', 'Perception fusion', 'Retrospective safety', 'Launch readiness', 'Release governance', 'Production hardening', 'ROTR (right-of-the-road)', ];