Deploy Sensorflow Studio

app_backend.py

@@ -1467,10 +1467,19 @@ app.include_router(labeleval_router)
 from sensorflow.megaeval.api import router as megaeval_router
 app.include_router(megaeval_router)
 
+# Sequential regression detection engine (anytime-valid, budgeted, paired).
+from sensorflow.seqeval.api import router as seqeval_router
+app.include_router(seqeval_router)
+
 # BEV-Fusion perception engine (camera+LiDAR fusion, masklet tracking, self-eval).
 from sensorflow.bevfusion.api import router as bevfusion_router
 app.include_router(bevfusion_router)
 
+# Safety & compliance layer (ODD coverage, release gates, SSAM, calibration,
+# discrepancy mining, scenario DB, semantic mining).
+from sensorflow.safety.api import router as safety_router
+app.include_router(safety_router)
+
 # Mount static folder
 static_dir = Path(__file__).parent / "static"
 if static_dir.exists():

sensorflow/safety/__init__.py

@@ -0,0 +1,21 @@
+"""Safety & compliance layer for Sensorflow Studio.
+
+Closes competitive feature gaps identified against AV perception evaluation
+platforms (Applied Intuition, Deepen AI, Scale Nucleus, Voxel51):
+
+- odd.py          ODD taxonomy + combinatorial coverage (ISO 34503 / ASAM OpenODD-inspired)
+- ssam_ext.py     Extended SSAM surrogate safety measures (FHWA SSAM: DRAC, DeltaS,
+                  MaxS, TTC/PET, Conflict Severity Index)
+- calibration.py  Multi-sensor extrinsic calibration validation (Deepen-style gate)
+- gates.py        Layered release gating + Safety Evidence Package (ISO 26262 /
+                  ISO 21448 SOTIF / UL 4600 mappings — the package SUPPORTS a
+                  safety case, it does not certify compliance)
+- discrepancy.py  Online-vs-offline auto-label discrepancy mining
+- scenario_db.py  Curated scenario database (Safety Pool-inspired, local)
+- semantic.py     Neuro-symbolic two-stage semantic scenario mining
+- api.py          FastAPI router exposing everything under /api/safety/*
+
+Foundation-model-style components are lightweight deterministic simulations
+behind clean interfaces, clearly marked as such — consistent with the rest of
+the platform.
+"""

sensorflow/safety/api.py

@@ -0,0 +1,414 @@
+"""FastAPI router for the safety & compliance layer (/api/safety/*).
+
+Endpoint map (frontend integration pass consumes this):
+
+ODD coverage (ISO 34503 / ASAM OpenODD-inspired)
+  GET  /api/safety/odd/taxonomy
+  GET  /api/safety/odd/coverage?run=...&dims=weather,lighting&include_cells=...
+  POST /api/safety/odd/fill-gap            {run, cell, num_sequences?, ...}
+
+Release gating + Safety Evidence Package (ISO 26262 / SOTIF / UL 4600-inspired)
+  POST /api/safety/gates/evaluate          {candidate_run, baseline_run, policy_overrides?}
+  GET  /api/safety/gates/policy
+  POST /api/safety/gates/policy            {overrides} (persisted)
+  GET  /api/safety/gates/result/{run_id}
+  GET  /api/safety/evidence/{run_id}?format=json|markdown
+
+Extended SSAM surrogate safety (FHWA SSAM)
+  POST /api/safety/ssam/analyze            {trajectories?|scenario?, params?, ...}
+  GET  /api/safety/ssam/summary?run=...
+
+Multi-sensor calibration validation (Deepen-style)
+  POST /api/safety/calibration/validate    {mode, rotation_offset_deg?, ...}
+  GET  /api/safety/calibration/status
+
+Auto-label discrepancy mining (continuous-learning loop)
+  POST /api/safety/discrepancy/mine        {dataset_id?, profile?, seed?}
+  GET  /api/safety/discrepancy/summary?dataset_id=...
+
+Scenario database (Safety Pool-inspired, local)
+  GET  /api/safety/scenarios?source=&severity=&scenario_type=&text=&limit=
+  POST /api/safety/scenarios/populate      (auto-import rare events)
+  POST /api/safety/scenarios/export        {filters?}
+
+Neuro-symbolic semantic mining
+  POST /api/safety/semantic-search         {concept, filters?, run?, target?, k?, use_llm?}
+
+Consensus extensions (additive evidence over the grader subsystem)
+  GET  /api/safety/consensus/summary?dataset_id=...
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+from fastapi import APIRouter, HTTPException, Query
+from pydantic import BaseModel, Field
+
+from sensorflow.safety import calibration as calib_mod
+from sensorflow.safety import discrepancy as disc_mod
+from sensorflow.safety import gates as gates_mod
+from sensorflow.safety import odd as odd_mod
+from sensorflow.safety import scenario_db as sdb_mod
+from sensorflow.safety import semantic as sem_mod
+from sensorflow.safety import ssam_ext
+
+router = APIRouter(prefix="/api/safety", tags=["safety"])
+
+
+def _mega_store():
+    from sensorflow.megaeval.runs import get_mega_store
+    return get_mega_store()
+
+
+def _eval_store():
+    from sensorflow.evaluation.records import get_store
+    return get_store()
+
+
+def _published_run(run_id: str):
+    run = _mega_store().runs.get(run_id)
+    if run is None:
+        raise HTTPException(404, f"Unknown evaluation run {run_id}")
+    if run.status != "published":
+        raise HTTPException(409, f"Run {run_id} is {run.status}; safety analyses "
+                                 "need a published run")
+    return run
+
+
+# ------------------------------------------------------------------ ODD
+
+
+@router.get("/odd/taxonomy")
+def odd_taxonomy():
+    return odd_mod.taxonomy()
+
+
+@router.get("/odd/coverage")
+def odd_coverage(run: str,
+                 dims: Optional[str] = Query(None, description="comma-separated"),
+                 min_samples: int = Query(odd_mod.DEFAULT_MIN_SAMPLES, ge=1, le=100_000),
+                 max_ci_width: float = Query(odd_mod.DEFAULT_MAX_CI_WIDTH, gt=0, le=1),
+                 target_recall: Optional[float] = Query(None, gt=0, le=1),
+                 include_cells: bool = False,
+                 max_gaps: int = Query(50, ge=1, le=500)):
+    r = _published_run(run)
+    dim_list = [d.strip() for d in dims.split(",") if d.strip()] if dims else None
+    try:
+        return odd_mod.coverage_for_run(
+            _mega_store(), r, dims=dim_list, min_samples=min_samples,
+            max_ci_width=max_ci_width, target_recall=target_recall,
+            include_cells=include_cells, max_gaps=max_gaps)
+    except ValueError as e:
+        raise HTTPException(422, str(e))
+
+
+class FillGapRequest(BaseModel):
+    run: str
+    cell: Dict[str, str]
+    num_sequences: int = Field(2, ge=1, le=10)
+    frames_per_sequence: int = Field(20, ge=5, le=60)
+    seed: Optional[int] = None
+
+
+@router.post("/odd/fill-gap")
+def odd_fill_gap(req: FillGapRequest):
+    r = _published_run(req.run)
+    try:
+        return odd_mod.fill_gap(_mega_store(), r, req.cell,
+                                num_sequences=req.num_sequences,
+                                frames_per_sequence=req.frames_per_sequence,
+                                seed=req.seed)
+    except ValueError as e:
+        raise HTTPException(422, str(e))
+
+
+# ------------------------------------------------------------------ gates
+
+
+class GateEvaluateRequest(BaseModel):
+    candidate_run: str
+    baseline_run: str
+    policy_overrides: Optional[Dict] = None
+
+
+@router.post("/gates/evaluate")
+def gates_evaluate(req: GateEvaluateRequest):
+    cand = _published_run(req.candidate_run)
+    base = _published_run(req.baseline_run)
+    return gates_mod.evaluate_gates(_mega_store(), cand, base,
+                                    policy_overrides=req.policy_overrides)
+
+
+@router.get("/gates/policy")
+def gates_policy():
+    return {"policy": gates_mod.get_policy(),
+            "defaults": gates_mod.DEFAULT_POLICY}
+
+
+class PolicyUpdateRequest(BaseModel):
+    overrides: Dict
+
+
+@router.post("/gates/policy")
+def gates_policy_update(req: PolicyUpdateRequest):
+    return {"policy": gates_mod.set_policy(req.overrides)}
+
+
+@router.get("/gates/result/{run_id}")
+def gates_result(run_id: str):
+    result = gates_mod.latest_gate_result(run_id)
+    if result is None:
+        raise HTTPException(404, f"No gate evaluation recorded for run {run_id}")
+    return result
+
+
+@router.get("/evidence/{run_id}")
+def evidence(run_id: str, format: str = Query("json", pattern="^(json|markdown)$")):
+    pkg = gates_mod.load_evidence(run_id)
+    if pkg is None:
+        raise HTTPException(404, f"No Safety Evidence Package for run {run_id}; "
+                                 "run POST /api/safety/gates/evaluate first")
+    if format == "markdown":
+        return {"run_id": run_id, "format": "markdown",
+                "markdown": gates_mod.render_markdown(pkg)}
+    return pkg
+
+
+# ------------------------------------------------------------------ SSAM
+
+
+class TrajectoryState(BaseModel):
+    t: float
+    x: float
+    y: float
+    speed: float
+    heading: float
+
+
+class Trajectory(BaseModel):
+    vehicle_id: str
+    vehicle_type: str = "car"
+    length: float = Field(4.5, gt=0)
+    width: float = Field(1.9, gt=0)
+    states: List[TrajectoryState]
+
+
+class SsamAnalyzeRequest(BaseModel):
+    trajectories: Optional[List[Trajectory]] = None
+    scenario: Optional[str] = Field(
+        None, description="crossing | rear_end | lane_change | mixed — generates "
+                          "synthetic TRJ-like trajectories when none are supplied")
+    seed: int = 0
+    reaction_delay_s: float = Field(0.6, ge=0, le=5)
+    params: Optional[Dict] = None
+
+
+@router.post("/ssam/analyze")
+def ssam_analyze(req: SsamAnalyzeRequest):
+    if req.trajectories:
+        trajs = [t.model_dump() for t in req.trajectories]
+        generated = None
+    else:
+        scenario = req.scenario or "mixed"
+        try:
+            trajs = ssam_ext.generate_trajectories(
+                seed=req.seed, scenario=scenario, reaction_delay_s=req.reaction_delay_s)
+        except ValueError as e:
+            raise HTTPException(422, str(e))
+        generated = {"scenario": scenario, "seed": req.seed,
+                     "reaction_delay_s": req.reaction_delay_s, "simulated": True}
+    result = ssam_ext.analyze_trajectories(trajs, params=req.params)
+    if generated:
+        result["generated"] = generated
+    return result
+
+
+@router.get("/ssam/summary")
+def ssam_summary(run: str, force: bool = False):
+    r = _published_run(run)
+    return ssam_ext.csi_for_run(r, force=force)
+
+
+# ------------------------------------------------------------------ calibration
+
+
+class CalibrationValidateRequest(BaseModel):
+    mode: str = Field("clean", pattern="^(clean|miscalibrated|perception_failure)$")
+    rotation_offset_deg: float = Field(2.0, ge=0, le=30)
+    translation_offset_m: float = Field(0.0, ge=0, le=5)
+    tamper_fraction: float = Field(0.25, ge=0, le=1)
+    num_objects: int = Field(14, ge=3, le=60)
+    seed: int = 3
+
+
+@router.post("/calibration/validate")
+def calibration_validate(req: CalibrationValidateRequest):
+    return calib_mod.run_validation(
+        mode=req.mode, rotation_offset_deg=req.rotation_offset_deg,
+        translation_offset_m=req.translation_offset_m,
+        tamper_fraction=req.tamper_fraction, num_objects=req.num_objects,
+        seed=req.seed)
+
+
+@router.get("/calibration/status")
+def calibration_status():
+    status = calib_mod.latest_status()
+    if status is None:
+        return {"status": "NEVER_RUN",
+                "note": "POST /api/safety/calibration/validate to run a validation"}
+    return status
+
+
+# ------------------------------------------------------------------ discrepancy mining
+
+
+class DiscrepancyMineRequest(BaseModel):
+    dataset_id: Optional[str] = Field(
+        None, description="defaults to the newest labeleval dataset")
+    profile: Optional[Dict] = None
+    seed: int = 101
+
+
+@router.post("/discrepancy/mine")
+def discrepancy_mine(req: DiscrepancyMineRequest):
+    store = _eval_store()
+    dataset_id = req.dataset_id
+    if dataset_id is None:
+        datasets = store.all("datasets")
+        if not datasets:
+            raise HTTPException(404, "No labeleval datasets exist; generate one first "
+                                     "(POST /api/labeleval/datasets/generate)")
+        dataset_id = max(datasets, key=lambda d: d.created_at).dataset_id
+    try:
+        report = disc_mod.mine(store, dataset_id, profile=req.profile, seed=req.seed)
+    except KeyError as e:
+        raise HTTPException(404, str(e))
+    # full per-discrepancy list can be large; cap the response (persisted intact)
+    slim = dict(report)
+    slim["discrepancies"] = report["discrepancies"][:100]
+    slim["discrepancies_truncated"] = len(report["discrepancies"]) > 100
+    return slim
+
+
+@router.get("/discrepancy/summary")
+def discrepancy_summary(dataset_id: Optional[str] = None):
+    if dataset_id:
+        summary = disc_mod.latest_summary(dataset_id)
+        if summary is None:
+            raise HTTPException(404, f"No discrepancy mining report for {dataset_id}")
+        return summary
+    store = _eval_store()
+    summaries = []
+    for ds in store.all("datasets"):
+        s = disc_mod.latest_summary(ds.dataset_id)
+        if s is not None:
+            summaries.append(s)
+    return {"datasets": summaries}
+
+
+# ------------------------------------------------------------------ scenario database
+
+
+@router.get("/scenarios")
+def scenarios(scenario_type: Optional[str] = None, source: Optional[str] = None,
+              severity: Optional[str] = None, weather: Optional[str] = None,
+              lighting: Optional[str] = None, text: Optional[str] = None,
+              limit: int = Query(100, ge=1, le=1000)):
+    odd_tags = {}
+    if weather:
+        odd_tags["weather"] = weather
+    if lighting:
+        odd_tags["lighting"] = lighting
+    db = sdb_mod.get_db()
+    records = db.search(scenario_type=scenario_type, source=source, severity=severity,
+                        odd_tags=odd_tags or None, text=text, limit=limit)
+    return {"counts": db.counts(),
+            "scenarios": [r.model_dump() for r in records]}
+
+
+@router.post("/scenarios/populate")
+def scenarios_populate():
+    added = sdb_mod.get_db().add_from_rare_events(_eval_store())
+    return {"imported_rare_events": added, "counts": sdb_mod.get_db().counts()}
+
+
+class ScenarioExportRequest(BaseModel):
+    scenario_type: Optional[str] = None
+    source: Optional[str] = None
+    severity: Optional[str] = None
+    text: Optional[str] = None
+    limit: int = Field(1000, ge=1, le=5000)
+
+
+@router.post("/scenarios/export")
+def scenarios_export(req: ScenarioExportRequest):
+    return sdb_mod.get_db().export_bundle(
+        scenario_type=req.scenario_type, source=req.source,
+        severity=req.severity, text=req.text, limit=req.limit)
+
+
+# ------------------------------------------------------------------ semantic search
+
+
+class SemanticSearchRequest(BaseModel):
+    concept: str = Field(..., min_length=2, max_length=500)
+    filters: Optional[Dict] = None
+    target: str = Field("containers", pattern="^(containers|scenarios)$")
+    run: Optional[str] = Field(None, description="megaeval run for target=containers; "
+                                                 "defaults to the newest published run")
+    k: int = Field(12, ge=1, le=100)
+    use_llm: Optional[bool] = Field(
+        None, description="None=auto (short-timeout Ollama attempt), true/false to force")
+
+
+@router.post("/semantic-search")
+def semantic_search(req: SemanticSearchRequest):
+    if req.target == "scenarios":
+        return sem_mod.search_scenarios(req.concept, filters=req.filters, k=req.k)
+    store = _mega_store()
+    if req.run:
+        run = _published_run(req.run)
+    else:
+        published = [r for r in store.runs.values() if r.status == "published"]
+        if not published:
+            raise HTTPException(404, "No published megaeval runs; start one via "
+                                     "/api/megaeval or use target=scenarios")
+        run = max(published, key=lambda r: r.created_at)
+    return sem_mod.search_containers(store, run, req.concept, filters=req.filters,
+                                     k=req.k, use_llm=req.use_llm)
+
+
+# ------------------------------------------------------------------ consensus evidence
+
+
+@router.get("/consensus/summary")
+def consensus_summary(dataset_id: str, examples: int = Query(5, ge=0, le=50)):
+    from sensorflow.evaluation.graders import dataset_grader_statistics
+    store = _eval_store()
+    if store.get("datasets", dataset_id) is None:
+        raise HTTPException(404, f"Unknown dataset {dataset_id}")
+    stats = dataset_grader_statistics(store, dataset_id)
+    if not stats:
+        raise HTTPException(404, f"No grader comparisons for {dataset_id}; run the "
+                                 "evaluation pipeline first")
+    anns = store.where("annotations", dataset_id=dataset_id)
+    sample = []
+    for a in sorted(anns, key=lambda x: x.annotation_id):
+        c = store.get("grader_comparisons", a.annotation_id)
+        if c is None:
+            continue
+        vector = {k[4:]: v for k, v in c.kappa_stats.items() if k.startswith("csv_")}
+        mbr = {k[4:]: v for k, v in c.kappa_stats.items() if k.startswith("mbr_")}
+        if not vector:
+            continue
+        sample.append({"annotation_id": a.annotation_id,
+                       "consensus_scalar": c.consensus,
+                       "consensus_score_vector": vector,
+                       "mbr": mbr})
+        if len(sample) >= examples:
+            break
+    return {"dataset_id": dataset_id, "statistics": stats,
+            "examples": sample,
+            "note": "grader panel is simulated (see graders.py); kappa/tau/MBR "
+                    "math is exact"}

sensorflow/safety/calibration.py

@@ -0,0 +1,302 @@
+"""Multi-sensor extrinsic calibration validation (Deepen-style safety gate).
+
+Models a 6-DoF extrinsic transform (rotation + translation) for a LiDAR ->
+camera sensor pair and validates it by projecting LiDAR points of labeled
+objects into the camera frame and measuring projection consistency against the
+objects' 2D boxes:
+
+- reprojection residual: the labeled 3D box corners are projected through the
+  assumed extrinsic; residual = projected-box center vs labeled 2D box center (px)
+- inlier ratio: fraction of the object's projected LiDAR points inside the
+  (padded) 2D box
+
+Diagnosis separates two failure modes:
+- MISCALIBRATED       systematic bias — residual vectors share a common
+                      direction/magnitude across objects (extrinsic drift);
+                      the rotation offset is estimated from the bias.
+- PERCEPTION_FAILURE  random disagreement — a minority of objects have large
+                      residuals in inconsistent directions while the global
+                      bias stays small (bad 2D detections, not calibration);
+                      flagged objects carry the platform's existing
+                      SENSOR_DISAGREEMENT failure reason.
+
+SIMULATED: scenes are synthetic (deterministic given seed). Calibrated scenes
+are generated with the true extrinsic; the validator then either uses a
+perturbed extrinsic (miscalibration injection) or tampers a fraction of the 2D
+boxes (perception-failure injection) to prove the detector works. Results feed
+the Scenario Quality Gate (gates.py) and the labeleval alerting store.
+"""
+
+from __future__ import annotations
+
+import math
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+import numpy as np
+
+from sensorflow.safety.store import read_json, write_json
+
+# Pinhole intrinsics matching the platform's synthetic camera (800x450, f=500).
+INTRINSICS = {"fx": 500.0, "fy": 500.0, "cx": 400.0, "cy": 225.0,
+              "width": 800, "height": 450}
+
+# True LiDAR->camera extrinsic. LiDAR frame: x fwd / y left / z up.
+# Camera frame: x right / y down / z fwd.  R0 maps axes; t0 is the lever arm (m).
+R0 = np.array([[0.0, -1.0, 0.0],
+               [0.0, 0.0, -1.0],
+               [1.0, 0.0, 0.0]])
+T0 = np.array([0.06, -0.12, 0.15])
+
+CLASS_DIMS = {  # l, w, h (m) — mirrors the labeleval synthetic priors
+    "vehicle": (4.5, 1.9, 1.6),
+    "truck": (8.0, 2.5, 3.1),
+    "pedestrian": (0.7, 0.7, 1.75),
+    "cyclist": (1.8, 0.6, 1.7),
+}
+
+DEFAULT_THRESHOLDS = {
+    "bias_px": 6.0,             # systematic residual => miscalibration suspect
+    "bias_to_scatter_ratio": 1.5,
+    "outlier_px": 14.0,         # per-object residual => perception suspect
+    "min_outlier_fraction": 0.10,
+    "min_inlier_ratio": 0.75,   # mean over objects for a healthy pair
+}
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def rotation_matrix(roll_deg: float = 0.0, pitch_deg: float = 0.0,
+                    yaw_deg: float = 0.0) -> np.ndarray:
+    """Intrinsic XYZ Euler rotation (applied in the camera frame)."""
+    r, p, y = (math.radians(a) for a in (roll_deg, pitch_deg, yaw_deg))
+    rx = np.array([[1, 0, 0], [0, math.cos(r), -math.sin(r)], [0, math.sin(r), math.cos(r)]])
+    ry = np.array([[math.cos(p), 0, math.sin(p)], [0, 1, 0], [-math.sin(p), 0, math.cos(p)]])
+    rz = np.array([[math.cos(y), -math.sin(y), 0], [math.sin(y), math.cos(y), 0], [0, 0, 1]])
+    return rz @ ry @ rx
+
+
+def make_extrinsic(rotation: Optional[np.ndarray] = None,
+                   translation: Optional[np.ndarray] = None) -> Dict:
+    return {"R": (rotation if rotation is not None else R0),
+            "t": (translation if translation is not None else T0)}
+
+
+def perturb_extrinsic(extrinsic: Dict, rotation_offset_deg: float = 0.0,
+                      axis: str = "pitch", translation_offset_m: float = 0.0) -> Dict:
+    kw = {f"{axis}_deg": rotation_offset_deg}
+    r_off = rotation_matrix(**kw)
+    t_off = np.array([translation_offset_m, 0.0, 0.0])
+    return {"R": r_off @ extrinsic["R"], "t": extrinsic["t"] + t_off}
+
+
+def project_points(points_lidar: np.ndarray, extrinsic: Dict) -> np.ndarray:
+    """Project Nx3 LiDAR points to Nx2 pixel coordinates."""
+    cam = points_lidar @ extrinsic["R"].T + extrinsic["t"]
+    z = np.clip(cam[:, 2], 0.1, None)
+    u = INTRINSICS["fx"] * cam[:, 0] / z + INTRINSICS["cx"]
+    v = INTRINSICS["fy"] * cam[:, 1] / z + INTRINSICS["cy"]
+    return np.stack([u, v], axis=1)
+
+
+# ------------------------------------------------------------------ scene generation
+
+
+def generate_scene(num_objects: int = 14, seed: int = 3) -> Dict:
+    """Deterministic calibrated scene: 3D objects with LiDAR point clusters and
+    2D boxes produced by projecting through the TRUE extrinsic."""
+    rng = np.random.default_rng(seed)
+    classes = list(CLASS_DIMS)
+    objects = []
+    attempts = 0
+    while len(objects) < num_objects and attempts < num_objects * 20:
+        attempts += 1
+        cls = classes[int(rng.integers(0, len(classes)))]
+        l, w, h = CLASS_DIMS[cls]
+        center = np.array([float(rng.uniform(8, 38)),
+                           float(rng.uniform(-9, 9)),
+                           h / 2])
+        n_pts = int(np.clip(240 * 12.0 / center[0], 40, 260))
+        pts = center + rng.uniform([-l / 2, -w / 2, -h / 2], [l / 2, w / 2, h / 2],
+                                   size=(n_pts, 3))
+        corners = center + np.array([[sx * l / 2, sy * w / 2, sz * h / 2]
+                                     for sx in (-1, 1) for sy in (-1, 1) for sz in (-1, 1)])
+        uv = project_points(corners, make_extrinsic())
+        u0, v0 = uv.min(axis=0)
+        u1, v1 = uv.max(axis=0)
+        if u0 < 0 or v0 < 0 or u1 > INTRINSICS["width"] or v1 > INTRINSICS["height"]:
+            continue  # keep only fully-visible objects
+        # 2D label noise: real camera labels are not pixel-perfect.
+        jitter = rng.normal(0.0, 1.2, size=4)
+        objects.append({
+            "object_id": f"cal-obj-{len(objects)}",
+            "class_name": cls,
+            "center_lidar": center.tolist(),
+            "corners_lidar": corners,
+            "points_lidar": pts,
+            "bbox_2d": [float(u0 + jitter[0]), float(v0 + jitter[1]),
+                        float(u1 - u0 + jitter[2]), float(v1 - v0 + jitter[3])],
+        })
+    return {"seed": seed, "objects": objects, "true_extrinsic": make_extrinsic(),
+            "simulated": True}
+
+
+# ------------------------------------------------------------------ validation
+
+
+def validate_scene(scene: Dict, assumed_extrinsic: Dict,
+                   thresholds: Optional[Dict] = None) -> Dict:
+    """Project each object's LiDAR points with the assumed extrinsic and measure
+    consistency against its 2D box; classify the failure mode."""
+    th = {**DEFAULT_THRESHOLDS, **(thresholds or {})}
+    per_object: List[Dict] = []
+    residuals = []
+    for obj in scene["objects"]:
+        corner_uv = project_points(np.asarray(obj["corners_lidar"]), assumed_extrinsic)
+        proj_center = (corner_uv.min(axis=0) + corner_uv.max(axis=0)) / 2
+        uv = project_points(np.asarray(obj["points_lidar"]), assumed_extrinsic)
+        bx, by, bw, bh = obj["bbox_2d"]
+        box_center = np.array([bx + bw / 2, by + bh / 2])
+        residual = proj_center - box_center
+        pad_x, pad_y = 0.15 * bw, 0.15 * bh
+        inside = ((uv[:, 0] >= bx - pad_x) & (uv[:, 0] <= bx + bw + pad_x)
+                  & (uv[:, 1] >= by - pad_y) & (uv[:, 1] <= by + bh + pad_y))
+        inlier_ratio = float(inside.mean())
+        r_mag = float(np.linalg.norm(residual))
+        residuals.append(residual)
+        per_object.append({
+            "object_id": obj["object_id"],
+            "class_name": obj["class_name"],
+            "residual_px": [round(float(residual[0]), 2), round(float(residual[1]), 2)],
+            "residual_magnitude_px": round(r_mag, 2),
+            "inlier_ratio": round(inlier_ratio, 4),
+            "flagged": False,
+            "failure_reason": None,
+        })
+
+    residuals = np.array(residuals)
+    mean_res = residuals.mean(axis=0)
+    bias = float(np.linalg.norm(mean_res))
+    scatter = float(np.linalg.norm(residuals - mean_res, axis=1).mean())
+    mean_inlier = float(np.mean([o["inlier_ratio"] for o in per_object]))
+    outliers = [o for o in per_object if o["residual_magnitude_px"] > th["outlier_px"]]
+    outlier_fraction = len(outliers) / max(len(per_object), 1)
+
+    systematic = bias > th["bias_px"] and bias > th["bias_to_scatter_ratio"] * scatter
+    if systematic:
+        status = "MISCALIBRATED"
+        for o in per_object:
+            o["flagged"] = True
+            o["failure_reason"] = "MISCALIBRATION"
+    elif outlier_fraction >= th["min_outlier_fraction"]:
+        status = "PERCEPTION_FAILURE"
+        for o in outliers:
+            o["flagged"] = True
+            # existing platform failure reason (records.FAILURE_REASONS)
+            o["failure_reason"] = "SENSOR_DISAGREEMENT"
+    else:
+        status = "CALIBRATED"
+
+    est_rot_deg = math.degrees(math.atan(bias / INTRINSICS["fx"]))
+    checks = [
+        {"name": "systematic_bias_px", "actual": round(bias, 2),
+         "threshold": th["bias_px"], "passed": bias <= th["bias_px"]},
+        {"name": "outlier_fraction", "actual": round(outlier_fraction, 4),
+         "threshold": th["min_outlier_fraction"],
+         "passed": outlier_fraction < th["min_outlier_fraction"]},
+        {"name": "mean_inlier_ratio", "actual": round(mean_inlier, 4),
+         "threshold": th["min_inlier_ratio"],
+         "passed": mean_inlier >= th["min_inlier_ratio"]},
+    ]
+    return {
+        "status": status,
+        "passed": status == "CALIBRATED",
+        "sensor_pair": "lidar_top->camera_front",
+        "checks": checks,
+        "metrics": {
+            "bias_px": round(bias, 3),
+            "bias_vector_px": [round(float(mean_res[0]), 3), round(float(mean_res[1]), 3)],
+            "scatter_px": round(scatter, 3),
+            "mean_inlier_ratio": round(mean_inlier, 4),
+            "outlier_fraction": round(outlier_fraction, 4),
+            "estimated_rotation_offset_deg": round(est_rot_deg, 3),
+            "num_objects": len(per_object),
+        },
+        "per_object": per_object,
+        "thresholds": th,
+        "diagnosis": {
+            "MISCALIBRATED": "systematic bias shared across objects — extrinsic drift",
+            "PERCEPTION_FAILURE": "random disagreement on a minority of objects — "
+                                  "detection defects, mapped to SENSOR_DISAGREEMENT",
+            "CALIBRATED": "projection consistent within thresholds",
+        }[status],
+        "simulated": True,
+    }
+
+
+def run_validation(mode: str = "clean", rotation_offset_deg: float = 2.0,
+                   translation_offset_m: float = 0.0, tamper_fraction: float = 0.25,
+                   num_objects: int = 14, seed: int = 3,
+                   thresholds: Optional[Dict] = None, persist: bool = True) -> Dict:
+    """Generate a calibrated scene, optionally inject a fault, and validate.
+
+    Modes: clean | miscalibrated (extrinsic perturbed by rotation_offset_deg)
+    | perception_failure (a fraction of 2D boxes tampered)."""
+    scene = generate_scene(num_objects=num_objects, seed=seed)
+    assumed = make_extrinsic()
+    if mode == "miscalibrated":
+        assumed = perturb_extrinsic(assumed, rotation_offset_deg=rotation_offset_deg,
+                                    axis="pitch", translation_offset_m=translation_offset_m)
+    elif mode == "perception_failure":
+        rng = np.random.default_rng(seed + 77)
+        n_tamper = max(1, int(round(tamper_fraction * len(scene["objects"]))))
+        idx = rng.choice(len(scene["objects"]), size=n_tamper, replace=False)
+        for i in idx:
+            box = scene["objects"][int(i)]["bbox_2d"]
+            box[0] += float(rng.choice([-1, 1]) * rng.uniform(25, 45))
+            box[1] += float(rng.choice([-1, 1]) * rng.uniform(20, 35))
+    elif mode != "clean":
+        raise ValueError(f"unknown mode {mode!r}")
+
+    result = validate_scene(scene, assumed, thresholds)
+    result["mode"] = mode
+    result["injected"] = {
+        "rotation_offset_deg": rotation_offset_deg if mode == "miscalibrated" else 0.0,
+        "translation_offset_m": translation_offset_m if mode == "miscalibrated" else 0.0,
+        "tamper_fraction": tamper_fraction if mode == "perception_failure" else 0.0,
+    }
+    result["seed"] = seed
+    result["created_at"] = _now()
+
+    if persist:
+        slim = {k: v for k, v in result.items() if k != "per_object"}
+        slim["per_object"] = [o for o in result["per_object"] if o["flagged"]][:25]
+        write_json(slim, "calibration", "status.json")
+        _emit_alert(result)
+    return result
+
+
+def latest_status() -> Optional[Dict]:
+    return read_json("calibration", "status.json")
+
+
+def _emit_alert(result: Dict) -> None:
+    """Surface failures in the existing labeleval alerting store (best-effort)."""
+    if result["status"] == "CALIBRATED":
+        return
+    try:
+        from sensorflow.evaluation.records import Alert, get_store, new_id
+        store = get_store()
+        store.put("alerts", Alert(
+            alert_id=new_id("alert"), kind="calibration",
+            severity="critical" if result["status"] == "MISCALIBRATED" else "warning",
+            message=(f"Calibration validation: {result['status']} — "
+                     f"bias {result['metrics']['bias_px']}px, "
+                     f"outliers {result['metrics']['outlier_fraction']*100:.0f}%, "
+                     f"{result['diagnosis']}"),
+            evidence_page="safety", evidence_id="calibration"))
+        store.save()
+    except Exception:
+        pass

sensorflow/safety/discrepancy.py

@@ -0,0 +1,247 @@
+"""Auto-labeling discrepancy mining (continuous-learning loop).
+
+Industry concept: production AV stacks run a weaker real-time ("online")
+perception pass in the vehicle and a stronger non-causal ("offline") auto-label
+pass over the same logs. Diffing the two is one of the highest-yield scenario
+mining signals — anything the online stack missed but the offline stack found
+is a candidate training example / regression scenario.
+
+This module simulates the online pass deterministically over an existing
+labeleval dataset (the stored auto-labels play the role of the strong offline
+pass), diffs the two, and converts discrepancies into queryable scenario-mining
+records:
+
+- MISSED_ONLINE:  offline labeled the object, online did not detect it
+- CLASS_MISMATCH: both detected it, classes differ
+- GEOMETRY_DELTA: both detected it, geometry disagrees (center offset / IoU)
+- OFFLINE_MISS:   online detected an object the offline pass dropped (rare)
+
+High-severity discrepancies feed the existing rare-event store; all of them
+feed the scenario database (source="discrepancy") and cohort summaries.
+
+SIMULATED: the online pass is a seeded degradation model over ground truth
+(higher miss rates, class confusion, geometry noise), not a real detector.
+"""
+
+from __future__ import annotations
+
+import math
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+import numpy as np
+
+from sensorflow.evaluation.records import (
+    SAFETY_CRITICAL_CLASSES,
+    EvalStore,
+    RareEvent,
+    new_id,
+)
+from sensorflow.metrics.perception_3d import bev_iou
+from sensorflow.safety.store import read_json, write_json
+
+DEFAULT_ONLINE_PROFILE = {
+    "extra_miss_rate": 0.12,        # online misses this fraction on top of offline
+    "miss_rate_by_class": {"pedestrian": 0.08, "cyclist": 0.08},  # additional VRU misses
+    "class_confusion_rate": 0.05,
+    "position_noise_m": 0.55,
+    "yaw_noise_deg": 6.0,
+    "night_miss_multiplier": 1.6,   # online degrades harder at night/rain
+    "rain_miss_multiplier": 1.4,
+}
+
+GEOMETRY_CENTER_THRESHOLD_M = 0.75
+GEOMETRY_IOU_THRESHOLD = 0.5
+
+_CONFUSABLE = {
+    "vehicle": "truck", "truck": "vehicle", "pedestrian": "cyclist",
+    "cyclist": "pedestrian", "motorcycle": "cyclist",
+}
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def simulate_online_pass(store: EvalStore, dataset_id: str,
+                         profile: Optional[Dict] = None, seed: int = 101) -> Dict[str, Dict]:
+    """Deterministic weaker 'online' perception pass over the dataset's GT.
+
+    Returns {gt_id: detection dict} for every ground-truth object the online
+    stack detected. SIMULATED degradation model, clearly marked.
+    """
+    p = {**DEFAULT_ONLINE_PROFILE, **(profile or {})}
+    rng = np.random.default_rng(seed)
+    detections: Dict[str, Dict] = {}
+    frames = sorted(store.where("frames", dataset_id=dataset_id), key=lambda f: f.index)
+    for frame in frames:
+        cond_mult = 1.0
+        if frame.time_of_day == "night":
+            cond_mult *= p["night_miss_multiplier"]
+        if frame.weather not in ("clear", ""):
+            cond_mult *= p["rain_miss_multiplier"]
+        for gt in frame.gt_boxes:
+            miss = p["extra_miss_rate"] + p["miss_rate_by_class"].get(gt.class_name, 0.0)
+            miss = min(1.0, miss * cond_mult) if miss < 1.0 else 1.0
+            if rng.random() < miss:
+                continue  # online miss
+            x, y, z, l, w, h, yaw = gt.bbox_3d
+            x += float(rng.normal(0, p["position_noise_m"]))
+            y += float(rng.normal(0, p["position_noise_m"]))
+            yaw += float(rng.normal(0, math.radians(p["yaw_noise_deg"])))
+            cls = gt.class_name
+            if rng.random() < p["class_confusion_rate"]:
+                cls = _CONFUSABLE.get(cls, cls)
+            detections[gt.gt_id] = {
+                "gt_id": gt.gt_id, "frame_id": frame.frame_id, "class_name": cls,
+                "bbox_3d": [round(v, 4) for v in (x, y, z, l, w, h, yaw)],
+                "confidence": round(float(rng.uniform(0.35, 0.85)), 3),
+            }
+    return detections
+
+
+def mine(store: EvalStore, dataset_id: str, profile: Optional[Dict] = None,
+         seed: int = 101, persist: bool = True) -> Dict:
+    """Diff the simulated online pass against the stored offline auto-labels."""
+    dataset = store.get("datasets", dataset_id)
+    if dataset is None:
+        raise KeyError(f"Unknown dataset {dataset_id}")
+
+    online = simulate_online_pass(store, dataset_id, profile, seed)
+    frames = {f.frame_id: f for f in store.where("frames", dataset_id=dataset_id)}
+    offline_by_gt = {a.matched_gt_id: a
+                     for a in store.where("annotations", dataset_id=dataset_id)
+                     if a.matched_gt_id}
+
+    discrepancies: List[Dict] = []
+    cohort_totals: Dict[str, int] = {}
+    cohort_hits: Dict[str, int] = {}
+
+    for frame in frames.values():
+        for gt in frame.gt_boxes:
+            cohort = f"{gt.class_name}/{frame.weather}/{frame.time_of_day}"
+            cohort_totals[cohort] = cohort_totals.get(cohort, 0) + 1
+            off = offline_by_gt.get(gt.gt_id)
+            on = online.get(gt.gt_id)
+            dtype = details = None
+            if off is not None and on is None:
+                dtype = "MISSED_ONLINE"
+                details = {"offline_confidence": off.confidence}
+            elif off is not None and on is not None:
+                if on["class_name"] != off.class_name:
+                    dtype = "CLASS_MISMATCH"
+                    details = {"offline_class": off.class_name,
+                               "online_class": on["class_name"]}
+                elif off.bbox_3d:
+                    center_d = math.hypot(on["bbox_3d"][0] - off.bbox_3d[0],
+                                          on["bbox_3d"][1] - off.bbox_3d[1])
+                    iou = bev_iou(on["bbox_3d"], off.bbox_3d)
+                    if center_d > GEOMETRY_CENTER_THRESHOLD_M or iou < GEOMETRY_IOU_THRESHOLD:
+                        dtype = "GEOMETRY_DELTA"
+                        details = {"center_delta_m": round(center_d, 3),
+                                   "bev_iou": round(iou, 4)}
+            elif off is None and on is not None:
+                dtype = "OFFLINE_MISS"
+                details = {"online_class": on["class_name"]}
+            if dtype is None:
+                continue
+            cohort_hits[cohort] = cohort_hits.get(cohort, 0) + 1
+            safety = gt.class_name in SAFETY_CRITICAL_CLASSES
+            severity = ("critical" if dtype == "MISSED_ONLINE" and safety
+                        else "high" if dtype == "MISSED_ONLINE" or safety
+                        else "medium")
+            discrepancies.append({
+                "discrepancy_id": new_id("disc"),
+                "dataset_id": dataset_id,
+                "frame_id": frame.frame_id,
+                "gt_id": gt.gt_id,
+                "type": dtype,
+                "class_name": gt.class_name,
+                "weather": frame.weather,
+                "time_of_day": frame.time_of_day,
+                "safety_critical": safety,
+                "severity": severity,
+                "details": details,
+            })
+
+    by_type: Dict[str, int] = {}
+    for d in discrepancies:
+        by_type[d["type"]] = by_type.get(d["type"], 0) + 1
+
+    cohorts = []
+    for cohort, total in sorted(cohort_totals.items()):
+        hits = cohort_hits.get(cohort, 0)
+        cls, weather, tod = cohort.split("/")
+        cohorts.append({"cohort": cohort, "class": cls, "weather": weather,
+                        "time_of_day": tod, "objects": total, "discrepancies": hits,
+                        "discrepancy_rate": round(hits / total, 4)})
+    cohorts.sort(key=lambda c: -c["discrepancy_rate"])
+
+    total_objects = sum(cohort_totals.values())
+    report = {
+        "dataset_id": dataset_id,
+        "seed": seed,
+        "created_at": _now(),
+        "online_profile": {**DEFAULT_ONLINE_PROFILE, **(profile or {})},
+        "totals": {
+            "objects": total_objects,
+            "online_detections": len(online),
+            "offline_annotations": len(offline_by_gt),
+            "discrepancies": len(discrepancies),
+            "discrepancy_rate": round(len(discrepancies) / max(total_objects, 1), 4),
+        },
+        "by_type": by_type,
+        "cohorts": cohorts,
+        "discrepancies": discrepancies,
+        "simulated": True,
+        "method": "deterministic simulated online pass vs stored offline auto-labels; "
+                  "diff over shared ground-truth ids",
+    }
+
+    if persist:
+        write_json(report, "discrepancy", f"{dataset_id}.json")
+        _feed_stores(store, report)
+    return report
+
+
+def _feed_stores(store: EvalStore, report: Dict) -> None:
+    """Feed high-severity discrepancies into the rare-event store, all of them
+    into the scenario database, and record process units (best-effort)."""
+    critical = [d for d in report["discrepancies"] if d["severity"] == "critical"]
+    for d in critical[:40]:
+        store.put("rare_events", RareEvent(
+            event_id=new_id("evt"),
+            dataset_id=d["dataset_id"],
+            scenario_type="online_perception_miss",
+            severity="critical",
+            rarity_score=0.85,
+            anomaly_score=0.0,
+            confidence=0.9,
+            evidence_frames=[d["frame_id"]],
+            sensor_evidence={"discrepancy": f"{d['type']} on {d['class_name']} "
+                                            f"({d['weather']}/{d['time_of_day']})"},
+            description=f"Online stack missed a {d['class_name']} the offline "
+                        f"auto-label pass found (simulated online pass)",
+        ))
+    try:
+        from sensorflow.evaluation.process_units import ProcessMeter
+        ProcessMeter(store, run_id=report["dataset_id"]).record(
+            "discrepancy_mining", report["totals"]["objects"], factor=0.3)
+    except Exception:
+        pass
+    store.save()
+    try:
+        from sensorflow.safety import scenario_db
+        scenario_db.get_db().add_from_discrepancies(report)
+    except Exception:
+        pass
+
+
+def latest_summary(dataset_id: str) -> Optional[Dict]:
+    """Persisted mining report without the raw per-object rows."""
+    report = read_json("discrepancy", f"{dataset_id}.json")
+    if report is None:
+        return None
+    slim = {k: v for k, v in report.items() if k != "discrepancies"}
+    slim["sample_discrepancies"] = report["discrepancies"][:25]
+    return slim

sensorflow/safety/gates.py

@@ -0,0 +1,522 @@
+"""Layered release gating + Safety Evidence Package.
+
+Industry concept: staged release gates for AV perception stacks, producing an
+auditable evidence trail structured after three families of standards:
+
+- ISO 26262 (functional safety): deterministic, repeatable verification —
+  geometric plausibility checks, calibration validation, regression testing
+  against a pinned baseline with pinned thresholds.
+- ISO 21448 (SOTIF): performance-limitation risk — ODD coverage of the
+  scenario space, under-covered cells, surrogate-safety (SSAM-style) analysis
+  of near-miss severity.
+- UL 4600 (safety case): the Release Readiness Gate compiles all gate
+  evidence, methodology and lineage into a claim/evidence-structured Safety
+  Evidence Package (JSON + rendered markdown).
+
+IMPORTANT WORDING: the package SUPPORTS a safety case by organizing
+verifiable evidence. It does NOT certify or demonstrate compliance with any
+standard, and several inputs are simulated (marked per-record).
+
+Gates (evaluated in order, block-on-fail; every gate always produces an
+evidence record even when an earlier gate blocked):
+
+1. scenario_quality  — geometric plausibility (evaluation.validation) +
+                       multi-sensor calibration validation (safety.calibration)
+2. coverage          — ODD combinatorial coverage thresholds (safety.odd)
+3. regression        — model-vs-model compare (megaeval.analysis.compare_runs,
+                       not duplicated here)
+4. safety            — surrogate-safety: aggregate CSI must not increase vs
+                       baseline beyond tolerance (safety.ssam_ext)
+5. release_readiness — all of the above + compiles the Safety Evidence Package
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+from sensorflow.safety import calibration as calib_mod
+from sensorflow.safety import odd as odd_mod
+from sensorflow.safety import ssam_ext
+from sensorflow.safety.store import read_json, write_json
+
+DEFAULT_POLICY: Dict = {
+    "scenario_quality": {
+        "enabled": True,
+        "min_geometric_pass_rate": 0.70,
+        "geometric_sample_size": 24,
+        "block_on_miscalibration": True,
+    },
+    "coverage": {
+        "enabled": True,
+        "dims": ["weather", "lighting", "road_type"],
+        "min_samples": 50,
+        "max_ci_width": 0.25,
+        "min_coverage_rate": 0.60,
+        "min_production_weighted_coverage": 0.80,
+    },
+    "regression": {
+        "enabled": True,
+        # thresholds delegated to megaeval.analysis.DEFAULT_PROMOTION_POLICY;
+        # entries here override it (e.g. max_recall_drop)
+        "overrides": {},
+    },
+    "safety": {
+        "enabled": True,
+        "max_csi_increase_ratio": 0.15,  # candidate CSI <= baseline * (1 + ratio)
+        "max_csi_absolute": None,        # optional hard ceiling
+    },
+}
+
+_POLICY_FILE = ("gate_policy.json",)
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _merge(base: Dict, override: Dict) -> Dict:
+    out = dict(base)
+    for k, v in (override or {}).items():
+        if isinstance(v, dict) and isinstance(out.get(k), dict):
+            out[k] = _merge(out[k], v)
+        else:
+            out[k] = v
+    return out
+
+
+def get_policy() -> Dict:
+    stored = read_json(*_POLICY_FILE) or {}
+    return _merge(DEFAULT_POLICY, stored)
+
+
+def set_policy(overrides: Dict) -> Dict:
+    stored = _merge(read_json(*_POLICY_FILE) or {}, overrides or {})
+    write_json(stored, *_POLICY_FILE)
+    return get_policy()
+
+
+def _check(name: str, actual, threshold, passed: bool, direction: str = "") -> Dict:
+    return {"check": name, "actual": actual, "threshold": threshold,
+            "direction": direction, "passed": bool(passed)}
+
+
+def _gate_record(gate: str, name: str, standard_refs: List[str], checks: List[Dict],
+                 evidence: Dict, skipped: bool = False, notes: str = "") -> Dict:
+    if skipped:
+        status = "SKIPPED"
+    else:
+        status = "PASS" if all(c["passed"] for c in checks) else "BLOCK"
+    return {"gate": gate, "name": name, "status": status,
+            "standard_refs": standard_refs, "checks": checks,
+            "evidence": evidence, "notes": notes, "evaluated_at": _now()}
+
+
+# ------------------------------------------------------------------ gate 1
+
+
+def scenario_quality_gate(policy: Dict, eval_store=None) -> Dict:
+    """Geometric plausibility sample + calibration validation.
+
+    Geometric part: validates a deterministic sample of auto-label annotations
+    from the newest labeleval dataset (generating a small probe dataset if the
+    store is empty) through the existing strict validation engine. Calibration
+    part: latest calibration validation status, refreshed with a clean run if
+    none exists.
+    """
+    p = policy["scenario_quality"]
+    if not p.get("enabled", True):
+        return _gate_record("scenario_quality", "Scenario Quality Gate",
+                            ["ISO 26262-style deterministic verification"],
+                            [], {}, skipped=True, notes="disabled by policy")
+
+    from sensorflow.evaluation import synthetic
+    from sensorflow.evaluation.records import get_store
+    from sensorflow.evaluation.validation import validate_annotation
+
+    store = eval_store or get_store()
+    datasets = store.all("datasets")
+    probe_generated = False
+    if datasets:
+        ds = max(datasets, key=lambda d: d.created_at)
+    else:
+        ds = synthetic.generate_dataset(store, name="gate-probe", num_sequences=2,
+                                        frames_per_sequence=6, seed=101)
+        synthetic.generate_labels(store, ds, seed_offset=1)
+        probe_generated = True
+
+    anns = sorted(store.where("annotations", dataset_id=ds.dataset_id),
+                  key=lambda a: a.annotation_id)
+    sample = anns[:max(1, int(p["geometric_sample_size"]))]
+    frames = {f.frame_id: f for f in store.where("frames", dataset_id=ds.dataset_id)}
+    points_cache: Dict[str, object] = {}
+    n_pass = 0
+    fail_hist: Dict[str, int] = {}
+    for ann in sample:
+        frame = frames.get(ann.frame_id)
+        if frame is None:
+            continue
+        if ann.frame_id not in points_cache:
+            points_cache[ann.frame_id] = synthetic.frame_points(store, frame)
+        res = validate_annotation(store, ann, frame, points=points_cache[ann.frame_id])
+        if res.passed:
+            n_pass += 1
+        else:
+            for c in res.checks:
+                if c.applicable and not c.passed:
+                    fail_hist[c.gate] = fail_hist.get(c.gate, 0) + 1
+    pass_rate = n_pass / max(len(sample), 1)
+
+    calib = calib_mod.latest_status()
+    if calib is None:
+        calib = calib_mod.run_validation(mode="clean", persist=True)
+    calib_ok = (calib["status"] != "MISCALIBRATED"
+                if p["block_on_miscalibration"] else True)
+
+    checks = [
+        _check("geometric_pass_rate", round(pass_rate, 4),
+               p["min_geometric_pass_rate"], pass_rate >= p["min_geometric_pass_rate"], ">="),
+        _check("calibration_status", calib["status"], "not MISCALIBRATED", calib_ok, "=="),
+    ]
+    return _gate_record(
+        "scenario_quality", "Scenario Quality Gate",
+        ["ISO 26262-style deterministic verification (geometric plausibility, "
+         "calibration validation)"],
+        checks,
+        evidence={
+            "dataset_id": ds.dataset_id,
+            "probe_dataset_generated": probe_generated,
+            "annotations_sampled": len(sample),
+            "annotations_passed": n_pass,
+            "failing_check_histogram": fail_hist,
+            "calibration": {"status": calib["status"], "metrics": calib.get("metrics"),
+                            "mode": calib.get("mode"), "diagnosis": calib.get("diagnosis")},
+        },
+        notes="geometric checks run on the strict validation engine; calibration "
+              "scenes are simulated (see safety.calibration)")
+
+
+# ------------------------------------------------------------------ gate 2
+
+
+def coverage_gate(mega_store, candidate, policy: Dict) -> Dict:
+    p = policy["coverage"]
+    if not p.get("enabled", True):
+        return _gate_record("coverage", "Coverage Gate",
+                            ["ISO 21448/SOTIF-style ODD coverage"],
+                            [], {}, skipped=True, notes="disabled by policy")
+    cov = odd_mod.coverage_for_run(
+        mega_store, candidate, dims=p["dims"], min_samples=p["min_samples"],
+        max_ci_width=p["max_ci_width"], max_gaps=10)
+    s = cov["summary"]
+    checks = [
+        _check("coverage_rate", s["coverage_rate"], p["min_coverage_rate"],
+               s["coverage_rate"] >= p["min_coverage_rate"], ">="),
+        _check("production_weighted_coverage", s["production_weighted_coverage"],
+               p["min_production_weighted_coverage"],
+               s["production_weighted_coverage"] >= p["min_production_weighted_coverage"],
+               ">="),
+    ]
+    return _gate_record(
+        "coverage", "Coverage Gate",
+        ["ISO 21448/SOTIF-style performance-limitation risk (ODD coverage per "
+         "ISO 34503-inspired taxonomy)"],
+        checks,
+        evidence={"dims": cov["dims"], "thresholds": cov["thresholds"],
+                  "summary": s, "top_gaps": cov["gaps"][:5],
+                  "fill_requests_available": len(cov["fill_requests"])},
+        notes="coverage over exact metric-cube counts; production shares use a "
+              "marginal-independence assumption")
+
+
+# ------------------------------------------------------------------ gate 3
+
+
+def regression_gate(mega_store, candidate, baseline, policy: Dict) -> Dict:
+    p = policy["regression"]
+    if not p.get("enabled", True):
+        return _gate_record("regression", "Regression Gate",
+                            ["ISO 26262-style regression verification"],
+                            [], {}, skipped=True, notes="disabled by policy")
+    from sensorflow.megaeval.analysis import compare_runs
+    cmp = compare_runs(mega_store, candidate, baseline, policy=p.get("overrides") or None)
+    ok = cmp["recommendation"] == "PROMOTE"
+    checks = [_check("compare_recommendation", cmp["recommendation"], "PROMOTE", ok, "==")]
+    return _gate_record(
+        "regression", "Regression Gate",
+        ["ISO 26262-style deterministic regression testing vs pinned baseline"],
+        checks,
+        evidence={"headline_deltas": cmp["headline_deltas"],
+                  "regressions": cmp["regressions"],
+                  "blockers": cmp["blockers"],
+                  "policy": cmp["policy"],
+                  "worst_cohorts": cmp["worst_cohorts"][:5]},
+        notes="delegates to megaeval model-vs-model compare (single source of truth)")
+
+
+# ------------------------------------------------------------------ gate 4
+
+
+def safety_gate(candidate, baseline, policy: Dict) -> Dict:
+    p = policy["safety"]
+    if not p.get("enabled", True):
+        return _gate_record("safety", "Safety Gate",
+                            ["ISO 21448/SOTIF-style surrogate safety"],
+                            [], {}, skipped=True, notes="disabled by policy")
+    cand = ssam_ext.csi_for_run(candidate)
+    base = ssam_ext.csi_for_run(baseline)
+    cand_csi, base_csi = cand["aggregate_csi"], base["aggregate_csi"]
+    limit = base_csi * (1.0 + p["max_csi_increase_ratio"])
+    increase_ratio = ((cand_csi - base_csi) / base_csi) if base_csi > 1e-9 else (
+        0.0 if cand_csi <= 1e-9 else float("inf"))
+    checks = [
+        _check("aggregate_csi_vs_baseline", round(cand_csi, 4),
+               round(limit, 4), cand_csi <= limit + 1e-9, "<="),
+    ]
+    if p.get("max_csi_absolute") is not None:
+        checks.append(_check("aggregate_csi_absolute", round(cand_csi, 4),
+                             p["max_csi_absolute"], cand_csi <= p["max_csi_absolute"], "<="))
+    return _gate_record(
+        "safety", "Safety Gate",
+        ["ISO 21448/SOTIF-style surrogate safety (FHWA SSAM: TTC/PET/DRAC/CSI)"],
+        checks,
+        evidence={
+            "candidate_csi": cand_csi, "baseline_csi": base_csi,
+            "csi_increase_ratio": (round(increase_ratio, 4)
+                                   if increase_ratio != float("inf") else "inf"),
+            "candidate_reaction_model": cand["reaction_model"],
+            "baseline_reaction_model": base["reaction_model"],
+            "candidate_min_ttc_s": cand["min_ttc_s"],
+            "candidate_scenarios": cand["scenarios"],
+        },
+        notes="CSI from a deterministic simulated scenario suite conditioned on the "
+              "model's perception profile (marked simulated); the SSAM formulas "
+              "themselves are real math")
+
+
+# ------------------------------------------------------------------ orchestration
+
+
+GATE_ORDER = ["scenario_quality", "coverage", "regression", "safety"]
+
+
+def evaluate_gates(mega_store, candidate, baseline,
+                   policy_overrides: Optional[Dict] = None, eval_store=None) -> Dict:
+    """Run all gates for candidate vs baseline; compile + persist the Safety
+    Evidence Package. Layered semantics: overall decision blocks on the first
+    failing gate, but every gate is still evaluated for evidence."""
+    policy = _merge(get_policy(), policy_overrides or {})
+    gates = [
+        scenario_quality_gate(policy, eval_store=eval_store),
+        coverage_gate(mega_store, candidate, policy),
+        regression_gate(mega_store, candidate, baseline, policy),
+        safety_gate(candidate, baseline, policy),
+    ]
+    blocking = [g["gate"] for g in gates if g["status"] == "BLOCK"]
+    release_ready = not blocking
+    readiness = _gate_record(
+        "release_readiness", "Release Readiness Gate",
+        ["UL 4600-style safety-case evidence compilation"],
+        [_check("all_gates_pass", f"{len(gates) - len(blocking)}/{len(gates)}",
+                f"{len(gates)}/{len(gates)}", release_ready, "==")],
+        evidence={"blocking_gates": blocking,
+                  "first_blocking_gate": blocking[0] if blocking else None},
+        notes="compiles the Safety Evidence Package below")
+    gates.append(readiness)
+
+    package = build_evidence_package(mega_store, candidate, baseline, policy, gates,
+                                     release_ready, blocking)
+    write_json(package, "evidence", f"{candidate.run_id}.json")
+    from sensorflow.safety.store import safety_path
+    md = render_markdown(package)
+    with open(safety_path("evidence", f"{candidate.run_id}.md"), "w") as f:
+        f.write(md)
+
+    result = {
+        "candidate_run_id": candidate.run_id,
+        "baseline_run_id": baseline.run_id,
+        "decision": "RELEASE_READY" if release_ready else "BLOCKED",
+        "blocking_gates": blocking,
+        "gates": gates,
+        "policy": policy,
+        "evidence_package_id": package["package_id"],
+        "evaluated_at": _now(),
+    }
+    write_json(result, "gates", f"{candidate.run_id}.json")
+    return result
+
+
+def latest_gate_result(run_id: str) -> Optional[Dict]:
+    return read_json("gates", f"{run_id}.json")
+
+
+def load_evidence(run_id: str) -> Optional[Dict]:
+    return read_json("evidence", f"{run_id}.json")
+
+
+# ------------------------------------------------------------------ evidence package
+
+
+DISCLAIMER = (
+    "This package organizes verifiable evidence to SUPPORT a safety case. It does "
+    "not certify, demonstrate or imply compliance with ISO 26262, ISO 21448, "
+    "UL 4600 or any other standard. Several evidence sources are simulated and "
+    "are marked as such on the individual records.")
+
+STANDARD_MAPPINGS = [
+    {"standard": "ISO 26262 (functional safety)",
+     "style": "deterministic, repeatable verification with pinned thresholds",
+     "evidence": ["gates.scenario_quality (geometric plausibility, calibration)",
+                  "gates.regression (model-vs-model compare vs pinned baseline)",
+                  "lineage (pinned dataset/model/label/metric/code versions + seed)"]},
+    {"standard": "ISO 21448 / SOTIF (safety of the intended functionality)",
+     "style": "performance-limitation risk in the operational domain",
+     "evidence": ["gates.coverage + odd_coverage_summary (under-covered ODD cells, "
+                  "risk-ranked gaps)",
+                  "gates.safety + surrogate_safety_analysis (SSAM-style near-miss "
+                  "severity, CSI vs baseline)"]},
+    {"standard": "UL 4600 (safety case for autonomous products)",
+     "style": "claim -> argument -> evidence structure",
+     "evidence": ["decision + gates[] (claims with thresholds, actuals and "
+                  "pass/block outcomes)",
+                  "sampling_methodology (statistical argument: stratified "
+                  "risk-weighted review, Wilson CIs)",
+                  "the package itself as the evidence index"]},
+]
+
+
+def build_evidence_package(mega_store, candidate, baseline, policy: Dict,
+                           gates: List[Dict], release_ready: bool,
+                           blocking: List[str]) -> Dict:
+    cov_gate = next(g for g in gates if g["gate"] == "coverage")
+    reg_gate = next(g for g in gates if g["gate"] == "regression")
+    saf_gate = next(g for g in gates if g["gate"] == "safety")
+
+    review = None
+    try:
+        review = mega_store.artifacts(candidate.run_id).get("review")
+    except Exception:
+        pass
+
+    supplementary: Dict = {}
+    try:
+        from sensorflow.safety.scenario_db import get_db
+        supplementary["scenario_db"] = get_db().counts()
+    except Exception:
+        pass
+    try:
+        cal = calib_mod.latest_status()
+        if cal:
+            supplementary["calibration_status"] = {
+                "status": cal["status"], "metrics": cal.get("metrics")}
+    except Exception:
+        pass
+
+    return {
+        "package_id": f"sep-{candidate.run_id}",
+        "package_format": "sensorflow-safety-evidence/v1",
+        "generated_at": _now(),
+        "disclaimer": DISCLAIMER,
+        "candidate": {"run_id": candidate.run_id, "model_version": candidate.model_version,
+                      "population_id": candidate.population_id,
+                      "headline": candidate.headline},
+        "baseline": {"run_id": baseline.run_id, "model_version": baseline.model_version,
+                     "headline": baseline.headline},
+        "decision": {"release_ready": release_ready, "blocking_gates": blocking},
+        "gates": gates,
+        "gate_policy": policy,
+        "odd_coverage_summary": cov_gate["evidence"],
+        "regression_analysis": reg_gate["evidence"],
+        "surrogate_safety_analysis": saf_gate["evidence"],
+        "sampling_methodology": {
+            "config": candidate.lineage.get("sampling_config"),
+            "confidence_intervals": "Wilson score intervals on per-cell and "
+                                    "review-sample proportions",
+            "review_artifact": ({"target_n": review.get("target_n"),
+                                 "achieved_n": review.get("achieved_n")}
+                                if isinstance(review, dict) else None),
+        },
+        "lineage": {"candidate": candidate.lineage, "baseline": baseline.lineage},
+        "standard_mappings": STANDARD_MAPPINGS,
+        "supplementary": supplementary,
+    }
+
+
+def render_markdown(pkg: Dict) -> str:
+    d = pkg["decision"]
+    lines = [
+        f"# Safety Evidence Package — {pkg['candidate']['model_version']}",
+        "",
+        f"Package `{pkg['package_id']}` generated {pkg['generated_at']}",
+        "",
+        f"> {pkg['disclaimer']}",
+        "",
+        f"**Decision: {'RELEASE READY' if d['release_ready'] else 'BLOCKED'}**"
+        + (f" — blocking gates: {', '.join(d['blocking_gates'])}" if d["blocking_gates"] else ""),
+        "",
+        f"Candidate `{pkg['candidate']['run_id']}` ({pkg['candidate']['model_version']}) "
+        f"vs baseline `{pkg['baseline']['run_id']}` ({pkg['baseline']['model_version']})",
+        "",
+        "## Gate results",
+        "",
+        "| Gate | Status | Checks (actual vs threshold) |",
+        "|---|---|---|",
+    ]
+    for g in pkg["gates"]:
+        checks = "; ".join(
+            f"{c['check']}: {c['actual']} {c['direction'] or 'vs'} {c['threshold']} "
+            f"({'ok' if c['passed'] else 'FAIL'})" for c in g["checks"]) or "—"
+        lines.append(f"| {g['name']} | {g['status']} | {checks} |")
+
+    s = pkg["odd_coverage_summary"].get("summary", {})
+    lines += [
+        "",
+        "## ODD coverage (SOTIF-style)",
+        "",
+        f"- dims: {', '.join(pkg['odd_coverage_summary'].get('dims', []))}",
+        f"- adequate cells: {s.get('adequate_cells')}/{s.get('total_cells')} "
+        f"(coverage rate {s.get('coverage_rate')})",
+        f"- production-weighted coverage: {s.get('production_weighted_coverage')}",
+        f"- gap cells: {s.get('gap_cells')}",
+        "",
+        "## Surrogate safety (SSAM-style)",
+        "",
+        f"- candidate aggregate CSI: {pkg['surrogate_safety_analysis'].get('candidate_csi')}",
+        f"- baseline aggregate CSI: {pkg['surrogate_safety_analysis'].get('baseline_csi')}",
+        f"- increase ratio: {pkg['surrogate_safety_analysis'].get('csi_increase_ratio')}",
+        f"- candidate min TTC: {pkg['surrogate_safety_analysis'].get('candidate_min_ttc_s')} s",
+        "",
+        "## Regression analysis",
+        "",
+    ]
+    for h in pkg["regression_analysis"].get("headline_deltas", []):
+        lines.append(f"- {h['metric']}: {h['baseline']} -> {h['candidate']} (Δ {h['delta']:+})")
+    blockers = pkg["regression_analysis"].get("blockers", [])
+    if blockers:
+        lines += ["", "Blockers:"] + [f"- {b}" for b in blockers]
+
+    lines += [
+        "",
+        "## Methodology",
+        "",
+        f"- sampling: {pkg['sampling_methodology'].get('config')}",
+        f"- CIs: {pkg['sampling_methodology'].get('confidence_intervals')}",
+        "",
+        "## Standard mappings (evidence organization, not compliance)",
+        "",
+    ]
+    for m in pkg["standard_mappings"]:
+        lines.append(f"### {m['standard']}")
+        lines.append(f"_{m['style']}_")
+        lines += [f"- {e}" for e in m["evidence"]]
+        lines.append("")
+
+    lines += [
+        "## Lineage",
+        "",
+        "```json",
+        __import__("json").dumps(pkg["lineage"], indent=2),
+        "```",
+        "",
+    ]
+    return "\n".join(lines)

sensorflow/safety/odd.py

@@ -0,0 +1,433 @@
+"""ODD coverage analysis (ISO 34503 / ASAM OpenODD-inspired).
+
+Industry concept mapping
+------------------------
+ISO 34503 defines an Operational Design Domain taxonomy in three top-level
+categories — scenery, environmental conditions, dynamic elements. ASAM OpenODD
+standardizes machine-readable ODD definitions. This module:
+
+1. exposes an ODD taxonomy built on the megaeval population dimensions (so
+   coverage is computed against the same vocabulary the metric cube uses),
+2. computes *combinatorial coverage* of the ODD space for an evaluation run:
+   which cells have data, how much, and with what quality (recall + Wilson CI),
+3. identifies statistically under-covered gaps (too few samples, or a recall
+   CI too wide to support any claim),
+4. ranks gaps by risk = production-like frequency x performance deficit,
+5. emits synthetic gap-filling scenario REQUESTS and can actually fulfil them
+   through the existing synthetic generator (sensorflow.evaluation.synthetic),
+   folding the generated objects back into coverage as a clearly-marked
+   synthetic supplement.
+
+Honest markers: the "production-like distribution" is the product of the
+population's per-dimension marginals (independence assumption, documented in
+the response); gap filling uses the deterministic synthetic generator, and the
+generator does not model road geometry — road_type on a supplement is a
+recorded request attribute, not simulated physics.
+"""
+
+from __future__ import annotations
+
+import itertools
+import math
+from datetime import datetime, timezone
+from typing import Dict, List, Optional, Tuple
+
+from sensorflow.megaeval import cube as cube_mod
+from sensorflow.megaeval import population as pop_mod
+from sensorflow.megaeval.population import DIMENSIONS, DIM_NAMES
+from sensorflow.megaeval.sampling import wilson_interval
+from sensorflow.safety.store import read_json, write_json
+
+DEFAULT_COVERAGE_DIMS = ["weather", "lighting", "road_type", "class"]
+DEFAULT_MIN_SAMPLES = 50
+DEFAULT_MAX_CI_WIDTH = 0.25
+
+# ISO 34503-style categorization of the dimensions we can instrument.
+ISO34503_CATEGORY = {
+    "road_type": "scenery",
+    "weather": "environmental_conditions",
+    "lighting": "environmental_conditions",
+    "occlusion": "environmental_conditions",
+    "class": "dynamic_elements",
+    "speed_band": "dynamic_elements",
+    "scenario": "dynamic_elements",
+    "distance_band": "dynamic_elements",
+    "sensor": "sensor_configuration",
+}
+
+# megaeval class vocabulary -> labeleval synthetic-generator class vocabulary.
+_CLASS_TO_GENERATOR = {
+    "vehicle": "vehicle",
+    "pedestrian": "pedestrian",
+    "cyclist": "cyclist",
+    "motorcycle": "motorcycle",
+    "truck": "truck",
+    "bus": "truck",  # generator has no bus prior; truck is the closest proxy
+}
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def taxonomy() -> Dict:
+    """Machine-readable ODD taxonomy (ISO 34503 categories, ASAM OpenODD-ish)."""
+    dims = {}
+    for name in DIM_NAMES:
+        dims[name] = {
+            "values": DIMENSIONS[name],
+            "iso34503_category": ISO34503_CATEGORY.get(name, "dynamic_elements"),
+            "instrumented": True,
+            "source": "megaeval population dimension",
+        }
+    # Declared but not instrumented in the current synthetic population.
+    dims["geography"] = {
+        "values": ["urban_california", "suburban_california", "highway_california",
+                   "rural_california"],
+        "iso34503_category": "scenery",
+        "instrumented": False,
+        "source": "placeholder — the current synthetic population does not carry "
+                  "a geography attribute; excluded from coverage computation",
+    }
+    return {
+        "standard_basis": "ISO 34503 ODD taxonomy structure (scenery / "
+                          "environmental conditions / dynamic elements), "
+                          "ASAM OpenODD-style machine-readable attributes",
+        "dimensions": dims,
+        "default_coverage_dims": DEFAULT_COVERAGE_DIMS,
+        "notes": [
+            "Coverage cells are the cartesian product of the selected dimensions.",
+            "Only instrumented dimensions may be used for coverage computation.",
+        ],
+    }
+
+
+# ------------------------------------------------------------------ core math
+
+
+def _marginals(dim_counts: Optional[Dict], cube_df, dims: List[str]) -> Dict[str, Dict[str, float]]:
+    """Per-dimension marginal probabilities of the production-like distribution.
+
+    Prefers the population dataset card (exact counts); falls back to the
+    cube's own n if no card is available (pure-cube unit tests).
+    """
+    out: Dict[str, Dict[str, float]] = {}
+    for dim in dims:
+        if dim_counts and dim in dim_counts:
+            counts = {v: float(c) for v, c in dim_counts[dim].items()}
+        else:
+            grp = cube_df.groupby(dim, observed=True)["n"].sum()
+            counts = {DIMENSIONS[dim][int(code)]: float(n) for code, n in grp.items()}
+        total = sum(counts.values()) or 1.0
+        out[dim] = {v: counts.get(v, 0.0) / total for v in DIMENSIONS[dim]}
+    return out
+
+
+def cell_id(cell: Dict[str, str], dims: List[str]) -> str:
+    return "|".join(f"{d}={cell[d]}" for d in dims)
+
+
+def compute_coverage(
+    cube_df,
+    dims: Optional[List[str]] = None,
+    dim_counts: Optional[Dict] = None,
+    min_samples: int = DEFAULT_MIN_SAMPLES,
+    max_ci_width: float = DEFAULT_MAX_CI_WIDTH,
+    target_recall: Optional[float] = None,
+    supplements: Optional[List[Dict]] = None,
+    max_gaps: int = 50,
+) -> Dict:
+    """Combinatorial ODD coverage from a metric cube (pure function, exact math).
+
+    Per cell: n (ground-truth objects), tp/fn, recall with Wilson CI,
+    adequacy (n >= min_samples AND CI width <= max_ci_width), production share
+    (product of marginals — independence assumption), performance deficit
+    (target_recall - Wilson lower bound, floored at 0) and
+    risk = production_share x deficit.
+    """
+    dims = [d for d in (dims or DEFAULT_COVERAGE_DIMS) if d in DIM_NAMES]
+    if not dims:
+        raise ValueError("no valid ODD dimensions selected")
+    n_cells = math.prod(len(DIMENSIONS[d]) for d in dims)
+    if n_cells > 2000:
+        raise ValueError(f"selected dimensions produce {n_cells} cells (max 2000); "
+                         "choose fewer dimensions")
+
+    rows, _ = cube_mod.aggregate(cube_df, None, dims, ["n", "tp", "fn", "recall", "mean_iou"],
+                                 limit=2000)
+    observed = {tuple(r[d] for d in dims): r for r in rows}
+
+    overall_rows, _ = cube_mod.aggregate(cube_df, None, None, ["recall"], 1)
+    overall_recall = overall_rows[0].get("recall") if overall_rows else None
+    if target_recall is None:
+        target_recall = overall_recall if overall_recall is not None else 0.9
+
+    marg = _marginals(dim_counts, cube_df, dims)
+
+    # Index synthetic supplements by cell id (only those matching these dims).
+    supp_by_cell: Dict[str, Dict] = {}
+    for s in supplements or []:
+        c = s.get("cell") or {}
+        if set(c.keys()) == set(dims):
+            cid = cell_id(c, dims)
+            agg = supp_by_cell.setdefault(cid, {"n_added": 0, "tp_added": 0, "fn_added": 0,
+                                                "datasets": []})
+            agg["n_added"] += int(s.get("n_added", 0))
+            agg["tp_added"] += int(s.get("tp_added", 0))
+            agg["fn_added"] += int(s.get("fn_added", 0))
+            if s.get("dataset_id"):
+                agg["datasets"].append(s["dataset_id"])
+
+    cells: List[Dict] = []
+    for combo in itertools.product(*(DIMENSIONS[d] for d in dims)):
+        cell = dict(zip(dims, combo))
+        cid = cell_id(cell, dims)
+        r = observed.get(combo, {})
+        n = int(r.get("n", 0))
+        tp = int(r.get("tp", 0))
+        fn = int(r.get("fn", 0))
+
+        supp = supp_by_cell.get(cid)
+        if supp:
+            n += supp["n_added"]
+            tp += supp["tp_added"]
+            fn += supp["fn_added"]
+
+        gt_n = tp + fn
+        recall = (tp / gt_n) if gt_n > 0 else None
+        lo, hi = wilson_interval(tp, gt_n)
+        ci_width = hi - lo
+        share = math.prod(marg[d].get(cell[d], 0.0) for d in dims)
+
+        reasons = []
+        if n < min_samples:
+            reasons.append("insufficient_samples")
+        if ci_width > max_ci_width:
+            reasons.append("ci_too_wide")
+        adequate = not reasons
+        deficit = max(0.0, float(target_recall) - lo)
+        risk = share * deficit
+
+        rec = {
+            "cell_id": cid,
+            "cell": cell,
+            "n": n,
+            "tp": tp,
+            "fn": fn,
+            "recall": None if recall is None else round(recall, 4),
+            "mean_iou": r.get("mean_iou"),
+            "wilson_ci": [round(lo, 4), round(hi, 4)],
+            "ci_width": round(ci_width, 4),
+            "production_share": round(share, 6),
+            "adequate": adequate,
+            "is_gap": not adequate,
+            "gap_reasons": reasons,
+            "performance_deficit": round(deficit, 4),
+            "risk": round(risk, 6),
+        }
+        if supp:
+            rec["synthetic_supplement"] = supp
+        cells.append(rec)
+
+    populated = [c for c in cells if c["n"] > 0]
+    adequate_cells = [c for c in cells if c["adequate"]]
+    gaps = sorted((c for c in cells if c["is_gap"]), key=lambda c: -c["risk"])
+
+    total_share = sum(c["production_share"] for c in cells) or 1.0
+    weighted_cov = sum(c["production_share"] for c in adequate_cells) / total_share
+
+    fill_requests = [gap_fill_request(g, min_samples) for g in gaps[:max_gaps]]
+
+    return {
+        "dims": dims,
+        "thresholds": {"min_samples": min_samples, "max_ci_width": max_ci_width,
+                       "target_recall": round(float(target_recall), 4)},
+        "summary": {
+            "total_cells": len(cells),
+            "populated_cells": len(populated),
+            "empty_cells": len(cells) - len(populated),
+            "adequate_cells": len(adequate_cells),
+            "gap_cells": len(gaps),
+            "coverage_rate": round(len(adequate_cells) / len(cells), 4),
+            "production_weighted_coverage": round(weighted_cov, 4),
+            "overall_recall": overall_recall,
+        },
+        "gaps": gaps[:max_gaps],
+        "fill_requests": fill_requests,
+        "cells": cells,
+        "method": "combinatorial cell coverage over the metric cube; recall CI = "
+                  "Wilson; production share = product of population marginals "
+                  "(independence assumption); risk = share x (target_recall - "
+                  "Wilson recall lower bound)",
+    }
+
+
+def gap_fill_request(gap: Dict, min_samples: int) -> Dict:
+    """A scenario REQUEST the synthetic generator can fulfil for one gap cell."""
+    cell = gap["cell"]
+    needed = max(0, min_samples - gap["n"])
+    cls = cell.get("class")
+    return {
+        "cell_id": gap["cell_id"],
+        "cell": cell,
+        "needed_samples": needed,
+        "risk": gap["risk"],
+        "generator": "sensorflow.evaluation.synthetic.generate_dataset",
+        "request": {
+            "weather": cell.get("weather"),
+            "time_of_day": ("night" if cell.get("lighting") == "night"
+                            else cell.get("lighting", "day")),
+            "target_class": _CLASS_TO_GENERATOR.get(cls) if cls else None,
+            "target_class_note": ("bus mapped to truck proxy" if cls == "bus" else None),
+            "road_type": cell.get("road_type"),
+            "road_type_note": "recorded request attribute; the synthetic generator "
+                              "does not model road geometry",
+            "num_sequences": 2,
+            "frames_per_sequence": max(10, min(40, needed)),
+        },
+    }
+
+
+# ------------------------------------------------------------------ run plumbing
+
+
+def _supplements_key(run_id: str) -> Tuple[str, str]:
+    return ("odd", f"{run_id}.json")
+
+
+def load_supplements(run_id: str) -> List[Dict]:
+    return read_json(*_supplements_key(run_id)) or []
+
+
+def coverage_for_run(store, run, dims: Optional[List[str]] = None,
+                     min_samples: int = DEFAULT_MIN_SAMPLES,
+                     max_ci_width: float = DEFAULT_MAX_CI_WIDTH,
+                     target_recall: Optional[float] = None,
+                     include_cells: bool = False,
+                     max_gaps: int = 50) -> Dict:
+    """ODD coverage for a published megaeval run (uses cube + dataset card)."""
+    art = store.artifacts(run.run_id)
+    meta = pop_mod.load_meta(run.population_id) or {}
+    result = compute_coverage(
+        art["cube"], dims=dims, dim_counts=meta.get("dim_counts"),
+        min_samples=min_samples, max_ci_width=max_ci_width,
+        target_recall=target_recall, supplements=load_supplements(run.run_id),
+        max_gaps=max_gaps)
+    result["run_id"] = run.run_id
+    result["population_id"] = run.population_id
+    result["model_version"] = run.model_version
+    if not include_cells:
+        result.pop("cells")
+    return result
+
+
+def fill_gap(store, run, cell: Dict[str, str], num_sequences: int = 2,
+             frames_per_sequence: int = 20, seed: Optional[int] = None) -> Dict:
+    """Generate targeted synthetic data for a gap cell and re-compute coverage.
+
+    Actually wired: calls the existing synthetic generator + label generator on
+    the labeleval store, retargets the generated frames' conditions to the gap
+    cell, measures how many ground-truth objects of the target class were added
+    and how many the simulated model labeled, persists the supplement, and
+    returns before/after coverage for that cell.
+    """
+    from sensorflow.evaluation import synthetic
+    from sensorflow.evaluation.process_units import ProcessMeter
+    from sensorflow.evaluation.records import get_store as get_legacy_store
+
+    dims = [d for d in DIM_NAMES if d in cell]
+    if not dims:
+        raise ValueError("cell must specify at least one known ODD dimension")
+    for d in dims:
+        if cell[d] not in DIMENSIONS[d]:
+            raise ValueError(f"unknown value {cell[d]!r} for dimension {d}")
+    cell = {d: cell[d] for d in dims}
+
+    before = coverage_for_run(store, run, dims=dims, include_cells=True)
+    cid = cell_id(cell, dims)
+    before_cell = next(c for c in before["cells"] if c["cell_id"] == cid)
+
+    legacy = get_legacy_store()
+    seed = seed if seed is not None else (run.seed + len(load_supplements(run.run_id))) % (2**31)
+    dataset = synthetic.generate_dataset(
+        legacy, name=f"odd-gap-{cid}", num_sequences=num_sequences,
+        frames_per_sequence=frames_per_sequence, seed=seed,
+        version="gap-fill", generated_from_model=run.model_version)
+
+    # Retarget conditions to the requested cell (deterministic post-adjustment;
+    # the generator itself does not take weather/lighting parameters).
+    weather = cell.get("weather")
+    lighting = cell.get("lighting")
+    for frame in legacy.where("frames", dataset_id=dataset.dataset_id):
+        if weather:
+            frame.weather = weather
+            if weather != "clear" and "adverse_weather" not in frame.scenario_tags:
+                frame.scenario_tags.append("adverse_weather")
+        if lighting:
+            frame.time_of_day = "night" if lighting == "night" else lighting
+            if lighting == "night" and "night_glare" not in frame.scenario_tags:
+                frame.scenario_tags.append("night_glare")
+        if "odd_gap_fill" not in frame.scenario_tags:
+            frame.scenario_tags.append("odd_gap_fill")
+        legacy.put("frames", frame)
+
+    annotations = synthetic.generate_labels(legacy, dataset,
+                                            model_version=run.model_version)
+    labeled_gt = {a.matched_gt_id for a in annotations if a.matched_gt_id}
+
+    target_class = _CLASS_TO_GENERATOR.get(cell.get("class", ""), None)
+    n_added = tp_added = 0
+    for frame in legacy.where("frames", dataset_id=dataset.dataset_id):
+        for gt in frame.gt_boxes:
+            if target_class is not None and gt.class_name != target_class:
+                continue
+            n_added += 1
+            if gt.gt_id in labeled_gt:
+                tp_added += 1
+    fn_added = n_added - tp_added
+
+    supplement = {
+        "cell_id": cid,
+        "cell": cell,
+        "dataset_id": dataset.dataset_id,
+        "n_added": n_added,
+        "tp_added": tp_added,
+        "fn_added": fn_added,
+        "seed": seed,
+        "source": "synthetic",
+        "created_at": _now(),
+        "note": "synthetic supplement generated by sensorflow.evaluation.synthetic; "
+                "counted separately from the immutable evaluation population",
+    }
+    supplements = load_supplements(run.run_id)
+    supplements.append(supplement)
+    write_json(supplements, *_supplements_key(run.run_id))
+
+    ProcessMeter(legacy, run_id=run.run_id).record("odd_gap_fill", n_added, factor=1.2)
+    legacy.audit("odd_gap_filled", "Dataset", dataset.dataset_id,
+                 f"cell={cid} n_added={n_added}")
+    legacy.save()
+
+    # Register the generated scenario in the scenario database.
+    try:
+        from sensorflow.safety import scenario_db
+        scenario_db.get_db().add_gap_fill(run.run_id, cell, dataset.dataset_id, n_added)
+    except Exception:
+        pass
+
+    after = coverage_for_run(store, run, dims=dims, include_cells=True)
+    after_cell = next(c for c in after["cells"] if c["cell_id"] == cid)
+    after.pop("cells")
+
+    return {
+        "run_id": run.run_id,
+        "cell_id": cid,
+        "cell": cell,
+        "generated_dataset_id": dataset.dataset_id,
+        "objects_added": n_added,
+        "labeled_added": tp_added,
+        "supplement": supplement,
+        "cell_before": before_cell,
+        "cell_after": after_cell,
+        "coverage_after": after,
+    }

sensorflow/safety/scenario_db.py

@@ -0,0 +1,223 @@
+"""Curated scenario database (Safety Pool-inspired, local).
+
+Industry concept: shared scenario databases (Safety Pool, StreetWise, ASAM
+OpenSCENARIO catalogs) curate reusable test scenarios with taxonomy tags, ODD
+attributes, severity and provenance. This module keeps a local, JSON-persisted
+equivalent under runs/safety/scenario_db.json:
+
+- record: id, scenario type taxonomy, ODD tags, source
+  (mined | synthetic | discrepancy | rare_event), severity, provenance /
+  lineage, evidence refs
+- auto-population from the existing rare-event store, discrepancy mining
+  reports and ODD gap-filling
+- search/filter + JSON bundle export
+
+Record ids are deterministic hashes of their provenance so repeated
+auto-population is idempotent.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import threading
+from datetime import datetime, timezone
+from typing import Dict, List, Literal, Optional
+
+from pydantic import BaseModel, Field
+
+from sensorflow.safety.store import read_json, write_json
+
+SourceType = Literal["mined", "synthetic", "discrepancy", "rare_event"]
+SEVERITIES = ["low", "medium", "high", "critical"]
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _det_id(*parts: str) -> str:
+    return "scn-" + hashlib.sha256("|".join(parts).encode()).hexdigest()[:12]
+
+
+class ScenarioRecord(BaseModel):
+    scenario_id: str
+    scenario_type: str
+    odd_tags: Dict[str, str] = Field(default_factory=dict)
+    source: SourceType = "mined"
+    severity: str = "medium"
+    description: str = ""
+    provenance: Dict[str, str] = Field(default_factory=dict)
+    evidence_refs: List[str] = Field(default_factory=list)
+    created_at: str = Field(default_factory=_now)
+
+
+class ScenarioDB:
+    """JSON-persisted scenario catalog with idempotent auto-population."""
+
+    def __init__(self):
+        self._lock = threading.RLock()
+        self._records: Dict[str, ScenarioRecord] = {}
+        self._load()
+
+    def _load(self) -> None:
+        raw = read_json("scenario_db.json") or {}
+        for sid, payload in raw.get("records", {}).items():
+            try:
+                self._records[sid] = ScenarioRecord.model_validate(payload)
+            except Exception:
+                continue
+
+    def _save(self) -> None:
+        write_json({"records": {sid: r.model_dump() for sid, r in self._records.items()},
+                    "updated_at": _now()}, "scenario_db.json")
+
+    # ---- ingestion
+
+    def add(self, record: ScenarioRecord) -> ScenarioRecord:
+        with self._lock:
+            self._records[record.scenario_id] = record
+            self._save()
+        return record
+
+    def add_from_rare_events(self, store) -> int:
+        """Idempotently import every rare event as a mined/rare_event scenario."""
+        added = 0
+        with self._lock:
+            for ev in store.all("rare_events"):
+                sid = _det_id("rare_event", ev.event_id)
+                if sid in self._records:
+                    continue
+                frame = store.get("frames", ev.evidence_frames[0]) if ev.evidence_frames else None
+                odd_tags = {}
+                if frame is not None:
+                    odd_tags = {"weather": frame.weather, "lighting": frame.time_of_day}
+                self._records[sid] = ScenarioRecord(
+                    scenario_id=sid,
+                    scenario_type=ev.scenario_type,
+                    odd_tags=odd_tags,
+                    source="rare_event",
+                    severity=ev.severity,
+                    description=ev.description,
+                    provenance={"event_id": ev.event_id, "dataset_id": ev.dataset_id},
+                    evidence_refs=[f"rare_event:{ev.event_id}"]
+                                  + [f"frame:{f}" for f in ev.evidence_frames[:5]],
+                )
+                added += 1
+            if added:
+                self._save()
+        return added
+
+    def add_from_discrepancies(self, report: Dict) -> int:
+        """Import discrepancy-mining findings (high/critical severity only)."""
+        added = 0
+        with self._lock:
+            for d in report.get("discrepancies", []):
+                if d["severity"] not in ("high", "critical"):
+                    continue
+                sid = _det_id("discrepancy", d["dataset_id"], d["gt_id"], d["type"])
+                if sid in self._records:
+                    continue
+                self._records[sid] = ScenarioRecord(
+                    scenario_id=sid,
+                    scenario_type=f"discrepancy_{d['type'].lower()}",
+                    odd_tags={"weather": d["weather"], "lighting": d["time_of_day"],
+                              "actor_class": d["class_name"]},
+                    source="discrepancy",
+                    severity=d["severity"],
+                    description=f"{d['type']} on {d['class_name']} "
+                                f"({d['weather']}/{d['time_of_day']}) — online vs "
+                                f"offline auto-label diff",
+                    provenance={"dataset_id": d["dataset_id"], "frame_id": d["frame_id"],
+                                "gt_id": d["gt_id"], "discrepancy_id": d["discrepancy_id"]},
+                    evidence_refs=[f"frame:{d['frame_id']}", f"gt:{d['gt_id']}"],
+                )
+                added += 1
+            if added:
+                self._save()
+        return added
+
+    def add_gap_fill(self, run_id: str, cell: Dict[str, str], dataset_id: str,
+                     n_objects: int) -> ScenarioRecord:
+        sid = _det_id("gap_fill", run_id, dataset_id)
+        rec = ScenarioRecord(
+            scenario_id=sid,
+            scenario_type="odd_gap_fill",
+            odd_tags={k: str(v) for k, v in cell.items()},
+            source="synthetic",
+            severity="medium",
+            description=f"Synthetic gap-filling scenario for ODD cell "
+                        f"{'|'.join(f'{k}={v}' for k, v in cell.items())} "
+                        f"({n_objects} objects)",
+            provenance={"run_id": run_id, "dataset_id": dataset_id},
+            evidence_refs=[f"dataset:{dataset_id}", f"odd_supplement:{run_id}"],
+        )
+        return self.add(rec)
+
+    # ---- queries
+
+    def search(self, scenario_type: Optional[str] = None, source: Optional[str] = None,
+               severity: Optional[str] = None, odd_tags: Optional[Dict[str, str]] = None,
+               text: Optional[str] = None, limit: int = 100) -> List[ScenarioRecord]:
+        with self._lock:
+            records = list(self._records.values())
+        out = []
+        for r in records:
+            if scenario_type and scenario_type.lower() not in r.scenario_type.lower():
+                continue
+            if source and r.source != source:
+                continue
+            if severity and r.severity != severity:
+                continue
+            if odd_tags and any(r.odd_tags.get(k) != v for k, v in odd_tags.items()):
+                continue
+            if text and text.lower() not in json.dumps(r.model_dump()).lower():
+                continue
+            out.append(r)
+        sev_rank = {s: i for i, s in enumerate(SEVERITIES)}
+        out.sort(key=lambda r: (-sev_rank.get(r.severity, 0), r.created_at))
+        return out[:max(1, min(limit, 1000))]
+
+    def counts(self) -> Dict:
+        with self._lock:
+            records = list(self._records.values())
+        by_source: Dict[str, int] = {}
+        by_severity: Dict[str, int] = {}
+        by_type: Dict[str, int] = {}
+        for r in records:
+            by_source[r.source] = by_source.get(r.source, 0) + 1
+            by_severity[r.severity] = by_severity.get(r.severity, 0) + 1
+            by_type[r.scenario_type] = by_type.get(r.scenario_type, 0) + 1
+        return {"total": len(records), "by_source": by_source,
+                "by_severity": by_severity, "by_type": by_type}
+
+    def export_bundle(self, **search_kwargs) -> Dict:
+        records = self.search(limit=search_kwargs.pop("limit", 1000), **search_kwargs)
+        return {
+            "bundle_format": "sensorflow-scenario-bundle/v1",
+            "exported_at": _now(),
+            "filters": {k: v for k, v in search_kwargs.items() if v},
+            "count": len(records),
+            "scenarios": [r.model_dump() for r in records],
+            "note": "local Safety Pool-inspired scenario catalog; scenario records "
+                    "reference platform evidence (frames, rare events, datasets)",
+        }
+
+
+_DB: Optional[ScenarioDB] = None
+_DB_LOCK = threading.Lock()
+
+
+def get_db() -> ScenarioDB:
+    global _DB
+    with _DB_LOCK:
+        if _DB is None:
+            _DB = ScenarioDB()
+        return _DB
+
+
+def reset_db() -> None:
+    """Test hook (also needed after set_safety_root)."""
+    global _DB
+    with _DB_LOCK:
+        _DB = None

sensorflow/safety/semantic.py

@@ -0,0 +1,340 @@
+"""Neuro-symbolic semantic scenario mining (lightweight, two-stage).
+
+Industry concept: natural-language scenario search over fleet data (Voxel51 /
+Scale Nucleus semantic search, Applied Intuition scenario mining). Two stages:
+
+1. SYMBOLIC filter — structured attributes (container ODD dimensions, error
+   rates, dynamics) restrict the candidate set exactly.
+2. REASONING scorer — if a local Ollama endpoint is reachable (reusing the
+   copilot plumbing in sensorflow/evaluation/copilot.py) it contributes a
+   natural-language rationale for the top candidates; otherwise (and always,
+   for the actual ranking) a DETERMINISTIC rule-based scorer maps concept
+   keywords to structured evidence via an explicit lexicon. Every result
+   carries per-stage explanations of why it matched.
+
+Hybrid retrieval: the rule score is blended with cosine similarity over the
+run's existing 32-d structural container embeddings (centroid of the top rule
+matches acts as the query anchor), combining symbolic and dense signals.
+
+Honest marker: the "reasoning" stage is a keyword lexicon over structured
+evidence unless Ollama is reachable; rankings are deterministic either way
+(the LLM adds rationale text, it never reorders results).
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+
+from sensorflow.megaeval import population as pop_mod
+from sensorflow.megaeval.population import DIMENSIONS
+
+# Each lexicon entry: (regex over the concept text, human-readable meaning,
+# scorer(row) -> (score in [0,1], evidence string)).
+_LEXICON: List[Tuple[str, str, object]] = []
+
+
+def _rule(pattern: str, meaning: str):
+    def deco(fn):
+        _LEXICON.append((pattern, meaning, fn))
+        return fn
+    return deco
+
+
+@_rule(r"pedestrian|vru|vulnerable|person|cyclist|bike", "vulnerable road users present")
+def _vru(row):
+    share = row["safety_n"] / max(row["n_objects"], 1)
+    return min(1.0, share * 4.0), f"safety-critical VRU share {share:.0%}"
+
+
+@_rule(r"night|dark|glare|low light", "night / low-illumination context")
+def _night(row):
+    if row["lighting"] == "night":
+        return 1.0, "lighting=night"
+    if row["lighting"] == "dusk":
+        return 0.5, "lighting=dusk"
+    return 0.0, ""
+
+
+@_rule(r"rain|wet|fog|snow|weather|storm", "adverse weather context")
+def _weather(row):
+    return (1.0, f"weather={row['weather']}") if row["weather"] != "clear" else (0.0, "")
+
+
+@_rule(r"hesitant|erratic|unusual|anomal|unpredict|jerky", "anomalous / erratic behavior evidence")
+def _anom(row):
+    rate = row["anomalies"] / max(row["n_objects"], 1)
+    return min(1.0, rate * 8.0), f"anomaly rate {rate:.0%}"
+
+
+@_rule(r"occlu|hidden|blocked|obstruct", "heavy occlusion context")
+def _occl(row):
+    return (1.0, "scenario=occlusion_heavy") if row["scenario"] == "occlusion_heavy" else (0.0, "")
+
+
+@_rule(r"near[ -]?miss|close call|conflict|collision|ttc", "near-miss / extreme-TTC scenario")
+def _nearmiss(row):
+    if row["scenario"] in ("near_miss", "extreme_ttc"):
+        return 1.0, f"scenario={row['scenario']}"
+    return 0.0, ""
+
+
+@_rule(r"intersection|junction|crossing", "intersection scenery")
+def _intersection(row):
+    return (1.0, "road_type=intersection") if row["road_type"] == "intersection" else (0.0, "")
+
+
+@_rule(r"urban|city|street", "urban scenery")
+def _urban(row):
+    return (1.0, "road_type=urban") if row["road_type"] == "urban" else (0.0, "")
+
+
+@_rule(r"highway|freeway|motorway", "highway scenery")
+def _highway(row):
+    return (1.0, "road_type=highway") if row["road_type"] == "highway" else (0.0, "")
+
+
+@_rule(r"rural|country", "rural scenery")
+def _rural(row):
+    return (1.0, "road_type=rural") if row["road_type"] == "rural" else (0.0, "")
+
+
+@_rule(r"lane edge|lane|curb|edge|merge", "lane-boundary interaction (proxy)")
+def _lane(row):
+    # Proxy: the population has no lane geometry; urban/intersection scenes are
+    # where lane-edge interactions concentrate. Explained as a proxy.
+    if row["road_type"] in ("urban", "intersection"):
+        return 0.5, f"road_type={row['road_type']} (lane-edge proxy — no lane geometry instrumented)"
+    return 0.0, ""
+
+
+@_rule(r"miss|fail|undetected|false negative|fn\b", "model misses (FN) evidence")
+def _fn(row):
+    rate = row["fn"] / max(row["n_objects"], 1)
+    return min(1.0, rate * 5.0), f"FN rate {rate:.0%}"
+
+
+@_rule(r"false positive|ghost|hallucinat|fp\b", "false-positive evidence")
+def _fp(row):
+    rate = row["fp"] / max(row["n_objects"], 1)
+    return min(1.0, rate * 5.0), f"FP rate {rate:.0%}"
+
+
+@_rule(r"sensor|degraded|dropout|noise", "sensor-degraded scenario")
+def _sensor(row):
+    return (1.0, "scenario=sensor_degraded") if row["scenario"] == "sensor_degraded" else (0.0, "")
+
+
+@_rule(r"risk|critical|danger|safety|severe", "high composite risk")
+def _risk(row):
+    return float(row["risk_score"]), f"risk_score {row['risk_score']:.2f}"
+
+
+def _container_rows(store, run) -> List[Dict]:
+    art = store.artifacts(run.run_id)
+    df = art["containers"]
+    rows = []
+    for _, r in df.iterrows():
+        rows.append({
+            "container_id": int(r["container_id"]),
+            **{dim: DIMENSIONS[dim][int(r[dim])] for dim in pop_mod.CONTAINER_DIMS},
+            "n_objects": int(r["n_objects"]),
+            "fn": int(r["fn"]), "fp": int(r["fp"]),
+            "anomalies": int(r["anomalies"]),
+            "safety_n": int(r["safety_n"]),
+            "risk_score": float(r["risk_score"]),
+        })
+    return rows
+
+
+def _symbolic_filter(rows: List[Dict], filters: Optional[Dict]) -> Tuple[List[Dict], Dict]:
+    before = len(rows)
+    applied = {}
+    for dim, values in (filters or {}).items():
+        if dim not in pop_mod.CONTAINER_DIMS:
+            continue
+        vals = values if isinstance(values, list) else [values]
+        rows = [r for r in rows if r[dim] in vals]
+        applied[dim] = vals
+    return rows, {"stage": "symbolic_filter", "applied_filters": applied,
+                  "candidates_before": before, "candidates_after": len(rows)}
+
+
+def _rule_score(concept: str, row: Dict) -> Tuple[float, List[Dict]]:
+    matched = []
+    total = 0.0
+    weight_sum = 0.0
+    for pattern, meaning, fn in _LEXICON:
+        if not re.search(pattern, concept, flags=re.IGNORECASE):
+            continue
+        score, evidence = fn(row)
+        weight_sum += 1.0
+        total += score
+        if score > 0:
+            matched.append({"term": pattern, "meaning": meaning,
+                            "evidence": evidence, "contribution": round(score, 4)})
+    if weight_sum == 0:
+        return float(row["risk_score"]), [{
+            "term": "(fallback)", "meaning": "no lexicon term matched the concept",
+            "evidence": f"ranked by composite risk_score {row['risk_score']:.2f}",
+            "contribution": round(float(row["risk_score"]), 4)}]
+    return total / weight_sum, matched
+
+
+def _llm_rationale(concept: str, top_rows: List[Dict], timeout_s: float = 2.0) -> Optional[Dict]:
+    """Optional Ollama pass (copilot plumbing); never reorders results."""
+    try:
+        import json as _json
+
+        import httpx
+
+        from sensorflow.evaluation.copilot import OLLAMA_ENDPOINTS
+        prompt = (
+            "You are a scenario-mining assistant. Concept query: "
+            f"{concept!r}. Candidate scenes (structured evidence):\n"
+            f"{_json.dumps(top_rows[:6], indent=1)[:2500]}\n"
+            "In 3 sentences, explain which candidates best match the concept and why."
+        )
+        for ep in OLLAMA_ENDPOINTS:
+            try:
+                res = httpx.post(ep["url"], json={
+                    "model": ep["model"],
+                    "messages": [{"role": "user", "content": prompt}],
+                    "stream": False,
+                }, timeout=timeout_s)
+                if res.status_code == 200:
+                    text = res.json().get("message", {}).get("content", "")
+                    if text:
+                        return {"provider": ep["url"], "rationale": text}
+            except Exception:
+                continue
+    except Exception:
+        pass
+    return None
+
+
+def search_containers(store, run, concept: str, filters: Optional[Dict] = None,
+                      k: int = 12, use_llm: Optional[bool] = None) -> Dict:
+    """Two-stage neuro-symbolic search over a run's containers.
+
+    Ranking = 0.65 * deterministic rule score + 0.35 * embedding similarity to
+    the centroid of the top rule matches (hybrid retrieval). LLM (if enabled
+    and reachable) adds rationale text only.
+    """
+    rows = _container_rows(store, run)
+    rows, stage1 = _symbolic_filter(rows, filters)
+
+    scored = []
+    for row in rows:
+        score, matched = _rule_score(concept, row)
+        scored.append((score, row, matched))
+    scored.sort(key=lambda t: (-t[0], t[1]["container_id"]))
+
+    # Hybrid: embedding similarity to the centroid of the top rule matches.
+    art = store.artifacts(run.run_id)
+    emb_ids, emb = art.get("emb_ids"), art.get("emb")
+    emb_sim: Dict[int, float] = {}
+    if emb is not None and scored:
+        anchor_ids = [t[1]["container_id"] for t in scored[:10] if t[0] > 0]
+        pos = np.where(np.isin(emb_ids, anchor_ids))[0]
+        if pos.size:
+            centroid = emb[pos].mean(axis=0)
+            norm = np.linalg.norm(centroid)
+            if norm > 1e-9:
+                sims = emb @ (centroid / norm)
+                emb_sim = {int(cid): float(s) for cid, s in zip(emb_ids, sims)}
+
+    results = []
+    for rule_score, row, matched in scored:
+        sim = emb_sim.get(row["container_id"], 0.0)
+        sim01 = (sim + 1.0) / 2.0
+        final = 0.65 * rule_score + 0.35 * sim01 if emb_sim else rule_score
+        results.append({
+            **row,
+            "score": round(final, 4),
+            "rule_score": round(rule_score, 4),
+            "embedding_similarity": round(sim, 4) if emb_sim else None,
+            "explanations": {
+                "stage1_symbolic": ("passed structured filters"
+                                    if stage1["applied_filters"] else "no filters applied"),
+                "stage2_reasoning": matched,
+            },
+        })
+    results.sort(key=lambda r: (-r["score"], r["container_id"]))
+    results = results[:max(1, min(k, 100))]
+
+    llm = None
+    if use_llm or use_llm is None:
+        # auto mode: short-timeout attempt; deterministic ranking is unaffected
+        if use_llm:  # explicit request gets a slightly longer timeout
+            llm = _llm_rationale(concept, results, timeout_s=6.0)
+        else:
+            llm = _llm_rationale(concept, results, timeout_s=1.5)
+
+    return {
+        "concept": concept,
+        "run_id": run.run_id,
+        "target": "containers",
+        "stage1": stage1,
+        "stage2": {
+            "stage": "reasoning_scorer",
+            "provider": (llm or {}).get("provider", "offline_deterministic"),
+            "note": "deterministic lexicon over structured evidence; LLM (when "
+                    "reachable) adds rationale text and never reorders results",
+        },
+        "hybrid": {"rule_weight": 0.65, "embedding_weight": 0.35,
+                   "embedding_source": "32-d structural container embeddings"},
+        "llm_rationale": (llm or {}).get("rationale"),
+        "results": results,
+    }
+
+
+def search_scenarios(concept: str, filters: Optional[Dict] = None, k: int = 12) -> Dict:
+    """Same two-stage idea over the scenario database (odd_tags as attributes)."""
+    from sensorflow.safety.scenario_db import get_db
+    records = get_db().search(limit=1000, odd_tags=None)
+    stage1_before = len(records)
+    filters = filters or {}
+    for key, val in filters.items():
+        vals = val if isinstance(val, list) else [val]
+        records = [r for r in records
+                   if r.odd_tags.get(key) in vals or getattr(r, key, None) in vals]
+
+    results = []
+    for r in records:
+        row = {  # adapt scenario record to the lexicon's row shape
+            "lighting": r.odd_tags.get("lighting", ""),
+            "weather": r.odd_tags.get("weather", ""),
+            "road_type": r.odd_tags.get("road_type", ""),
+            "scenario": r.scenario_type,
+            "n_objects": 1, "fn": 0, "fp": 0, "anomalies": 0,
+            "safety_n": 1 if r.odd_tags.get("actor_class") in
+                             ("pedestrian", "cyclist", "motorcycle") else 0,
+            "risk_score": {"low": 0.2, "medium": 0.45, "high": 0.7,
+                           "critical": 0.95}.get(r.severity, 0.4),
+        }
+        score, matched = _rule_score(concept, row)
+        text_bonus = 0.15 if re.search(r"\w", concept) and any(
+            w in r.description.lower() for w in re.findall(r"[a-z]{4,}", concept.lower())) else 0.0
+        results.append({
+            "scenario_id": r.scenario_id,
+            "scenario_type": r.scenario_type,
+            "source": r.source,
+            "severity": r.severity,
+            "odd_tags": r.odd_tags,
+            "description": r.description,
+            "score": round(min(1.0, score + text_bonus), 4),
+            "explanations": {"stage1_symbolic": filters or "no filters applied",
+                             "stage2_reasoning": matched},
+        })
+    results.sort(key=lambda x: (-x["score"], x["scenario_id"]))
+    return {
+        "concept": concept,
+        "target": "scenarios",
+        "stage1": {"stage": "symbolic_filter", "applied_filters": filters,
+                   "candidates_before": stage1_before, "candidates_after": len(results)},
+        "stage2": {"stage": "reasoning_scorer", "provider": "offline_deterministic"},
+        "results": results[:max(1, min(k, 100))],
+    }

sensorflow/safety/ssam_ext.py

@@ -0,0 +1,525 @@
+"""Extended SSAM surrogate safety measures (FHWA Surrogate Safety Assessment Model).
+
+Extends the platform's existing TTC/PET/severity SSAM logic (app_backend
+/api/ssam/*, kept intact) into a shared analysis module operating on TRJ-like
+trajectory data:
+
+- TTC via conflict-point projection: each vehicle is a rectangle polygon
+  projected along its trajectory (constant-velocity extrapolation) over a
+  look-ahead interval; TTC is the first projected time at which the two
+  rectangles' separation reaches zero.
+- PET via a zone-grid proximity function: actual footprints are rasterized
+  into a spatial grid; PET is the minimum positive time gap between one road
+  user leaving a cell and another entering it (classic SSAM definition).
+- DRAC (deceleration rate to avoid crash) = closing_speed^2 / (2 * gap).
+- DeltaS = max relative speed differential during a conflict (severity proxy).
+- MaxS   = max absolute speed of either participant during a conflict.
+- Collision probability over the look-ahead horizon:
+  p(t) = clip(1 - TTC(t)/TTC_threshold, 0, 1); the conflict-level probability
+  is the max instantaneous value, with the time integral reported as exposure.
+- CSI (Conflict Severity Index, time-weighted): per conflict,
+  CSI = sum_t [ 0.5 * mu * DeltaS(t)^2 * p(t) * dt ]
+  where mu is the reduced mass proxy of the pair (kinetic-energy proxy of the
+  relative motion). Aggregate CSI for a scenario suite is the sum across
+  conflicts; the release Safety Gate blocks candidates whose aggregate CSI
+  rises vs baseline beyond tolerance.
+
+Honest markers: trajectories are synthetic (generate_trajectories) and the
+model-conditioned reaction-time mapping in csi_for_run is a deterministic
+simulation of how perception quality affects downstream conflict severity —
+it is not a physics-validated driver model.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+
+from sensorflow.metrics.perception_3d import bev_iou
+from sensorflow.safety.store import read_json, write_json
+
+DEFAULT_PARAMS = {
+    "ttc_threshold_s": 1.5,      # FHWA SSAM default TTC conflict threshold
+    "pet_threshold_s": 5.0,      # FHWA SSAM default PET conflict threshold
+    "look_ahead_s": 3.0,
+    "projection_dt_s": 0.05,
+    "grid_cell_m": 0.5,
+    "rear_end_angle_deg": 30.0,  # SSAM conflict-type angle conventions
+    "crossing_angle_deg": 85.0,
+}
+
+# Unit-less mass proxies for the kinetic-energy severity term.
+MASS_PROXY = {
+    "car": 1.0, "vehicle": 1.0, "truck": 2.5, "bus": 2.2,
+    "motorcycle": 0.25, "cyclist": 0.10, "pedestrian": 0.08,
+}
+
+
+# ------------------------------------------------------------------ geometry
+
+
+def _velocity(state: Dict) -> Tuple[float, float]:
+    return (state["speed"] * math.cos(state["heading"]),
+            state["speed"] * math.sin(state["heading"]))
+
+
+def _support_extent(length: float, width: float, heading: float,
+                    line_angle: float) -> float:
+    """Half-extent of a rectangle along a line direction (support function)."""
+    phi = line_angle - heading
+    return (length / 2) * abs(math.cos(phi)) + (width / 2) * abs(math.sin(phi))
+
+
+def rect_gap(pa: Tuple[float, float], ha: float, la: float, wa: float,
+             pb: Tuple[float, float], hb: float, lb: float, wb: float) -> float:
+    """Separation between two rectangles along the center line (support-function
+    approximation; exact for aligned rear-end geometry, conservative otherwise)."""
+    dx, dy = pb[0] - pa[0], pb[1] - pa[1]
+    dist = math.hypot(dx, dy)
+    if dist < 1e-9:
+        return 0.0
+    ang = math.atan2(dy, dx)
+    gap = dist - _support_extent(la, wa, ha, ang) - _support_extent(lb, wb, hb, ang)
+    return max(0.0, gap)
+
+
+def rects_overlap(pa, ha, la, wa, pb, hb, lb, wb) -> bool:
+    """Exact rotated-rectangle overlap via the shared BEV IoU clipper."""
+    la, wa = max(la, 0.05), max(wa, 0.05)
+    lb, wb = max(lb, 0.05), max(wb, 0.05)
+    return bev_iou([pa[0], pa[1], 0, la, wa, 0, ha],
+                   [pb[0], pb[1], 0, lb, wb, 0, hb]) > 1e-12
+
+
+def projected_ttc(state_a: Dict, dims_a: Tuple[float, float],
+                  state_b: Dict, dims_b: Tuple[float, float],
+                  look_ahead: float, dt: float) -> Optional[float]:
+    """Conflict-point projection TTC: first projected time (constant-velocity
+    rectangle projection) at which the separation reaches zero.
+
+    Scans the look-ahead interval and linearly interpolates the zero crossing
+    of the gap, so linear closing geometry yields near-exact TTC values.
+    """
+    vax, vay = _velocity(state_a)
+    vbx, vby = _velocity(state_b)
+    la, wa = dims_a
+    lb, wb = dims_b
+    prev_tau, prev_gap = 0.0, None
+    tau = 0.0
+    while tau <= look_ahead + 1e-9:
+        pa = (state_a["x"] + vax * tau, state_a["y"] + vay * tau)
+        pb = (state_b["x"] + vbx * tau, state_b["y"] + vby * tau)
+        gap = rect_gap(pa, state_a["heading"], la, wa, pb, state_b["heading"], lb, wb)
+        if gap <= 1e-9 or rects_overlap(pa, state_a["heading"], la, wa,
+                                        pb, state_b["heading"], lb, wb):
+            if prev_gap is None or prev_gap <= 1e-9:
+                return tau
+            frac = prev_gap / max(prev_gap - gap, 1e-9)
+            return prev_tau + frac * (tau - prev_tau)
+        prev_tau, prev_gap = tau, gap
+        tau += dt
+    return None
+
+
+def drac(state_a: Dict, dims_a: Tuple[float, float],
+         state_b: Dict, dims_b: Tuple[float, float]) -> Optional[float]:
+    """Deceleration Rate to Avoid Crash = closing_speed^2 / (2 * gap)."""
+    pa, pb = (state_a["x"], state_a["y"]), (state_b["x"], state_b["y"])
+    dx, dy = pb[0] - pa[0], pb[1] - pa[1]
+    dist = math.hypot(dx, dy)
+    if dist < 1e-9:
+        return None
+    vax, vay = _velocity(state_a)
+    vbx, vby = _velocity(state_b)
+    # d|pb-pa|/dt ; negative when the pair is closing.
+    range_rate = ((dx * (vbx - vax)) + (dy * (vby - vay))) / dist
+    closing = -range_rate
+    if closing <= 0:
+        return None
+    gap = rect_gap(pa, state_a["heading"], *dims_a, pb, state_b["heading"], *dims_b)
+    return closing ** 2 / (2 * max(gap, 0.1))
+
+
+def collision_probability(ttc: Optional[float], ttc_threshold: float) -> float:
+    """Instantaneous collision probability proxy: 1 - TTC/threshold, clipped."""
+    if ttc is None:
+        return 0.0
+    return float(np.clip(1.0 - ttc / ttc_threshold, 0.0, 1.0))
+
+
+def _conflict_type(heading_a: float, heading_b: float, params: Dict) -> str:
+    diff = abs(heading_a - heading_b) % (2 * math.pi)
+    diff = min(diff, 2 * math.pi - diff)
+    deg = math.degrees(diff)
+    if deg < params["rear_end_angle_deg"]:
+        return "rear_end"
+    if deg > params["crossing_angle_deg"]:
+        return "crossing"
+    return "lane_change"
+
+
+# ------------------------------------------------------------------ zone-grid PET
+
+
+def _footprint_cells(state: Dict, length: float, width: float, cell: float) -> List[Tuple[int, int]]:
+    """Grid cells whose centers fall inside the vehicle rectangle footprint."""
+    x, y, h = state["x"], state["y"], state["heading"]
+    half_diag = math.hypot(length, width) / 2 + cell
+    cells = []
+    i0 = int(math.floor((x - half_diag) / cell))
+    i1 = int(math.ceil((x + half_diag) / cell))
+    j0 = int(math.floor((y - half_diag) / cell))
+    j1 = int(math.ceil((y + half_diag) / cell))
+    c, s = math.cos(-h), math.sin(-h)
+    hl, hw = max(length / 2, cell / 2), max(width / 2, cell / 2)
+    for i in range(i0, i1 + 1):
+        for j in range(j0, j1 + 1):
+            cx, cy = (i + 0.5) * cell - x, (j + 0.5) * cell - y
+            rx = cx * c - cy * s
+            ry = cx * s + cy * c
+            if abs(rx) <= hl and abs(ry) <= hw:
+                cells.append((i, j))
+    return cells
+
+
+def zone_grid_pet(traj_a: Dict, traj_b: Dict, cell: float) -> Optional[float]:
+    """Post-encroachment time via zone-grid occupancy intervals.
+
+    PET = min over shared cells of the positive gap between one vehicle leaving
+    the cell and the other entering it.
+    """
+    def occupancy(traj) -> Dict[Tuple[int, int], List[float]]:
+        occ: Dict[Tuple[int, int], List[float]] = {}
+        for st in traj["states"]:
+            for c in _footprint_cells(st, traj.get("length", 0.0),
+                                      traj.get("width", 0.0), cell):
+                span = occ.setdefault(c, [st["t"], st["t"]])
+                span[0] = min(span[0], st["t"])
+                span[1] = max(span[1], st["t"])
+        return occ
+
+    occ_a, occ_b = occupancy(traj_a), occupancy(traj_b)
+    pets = []
+    for c in set(occ_a) & set(occ_b):
+        (a0, a1), (b0, b1) = occ_a[c], occ_b[c]
+        if b0 > a1:
+            pets.append(b0 - a1)
+        elif a0 > b1:
+            pets.append(a0 - b1)
+        # overlapping occupancy => simultaneous presence (PET 0 handled by TTC path)
+        else:
+            pets.append(0.0)
+    return min(pets) if pets else None
+
+
+# ------------------------------------------------------------------ analysis
+
+
+def analyze_trajectories(trajectories: List[Dict], params: Optional[Dict] = None) -> Dict:
+    """Full SSAM-extended analysis: conflicts + surrogate measures + CSI."""
+    p = {**DEFAULT_PARAMS, **(params or {})}
+    conflicts: List[Dict] = []
+    total_time = 0.0
+    for traj in trajectories:
+        ts = [s["t"] for s in traj["states"]]
+        if len(ts) >= 2:
+            total_time = max(total_time, ts[-1] - ts[0])
+
+    for i in range(len(trajectories)):
+        for j in range(i + 1, len(trajectories)):
+            conflicts.extend(_analyze_pair(trajectories[i], trajectories[j], p))
+
+    agg = {
+        "num_conflicts": len(conflicts),
+        "by_type": {},
+        "min_ttc_s": None,
+        "min_pet_s": None,
+        "max_drac_mps2": None,
+        "max_delta_s_mps": None,
+        "max_s_mps": None,
+        "max_collision_probability": None,
+        "aggregate_csi": round(sum(c["csi"] for c in conflicts), 6),
+        "mean_csi_per_conflict": (round(sum(c["csi"] for c in conflicts) / len(conflicts), 6)
+                                  if conflicts else 0.0),
+        "observed_duration_s": round(total_time, 3),
+    }
+    for c in conflicts:
+        agg["by_type"][c["conflict_type"]] = agg["by_type"].get(c["conflict_type"], 0) + 1
+        for key, src, fn in (("min_ttc_s", "min_ttc_s", min), ("min_pet_s", "pet_s", min),
+                             ("max_drac_mps2", "max_drac_mps2", max),
+                             ("max_delta_s_mps", "delta_s_mps", max),
+                             ("max_s_mps", "max_s_mps", max),
+                             ("max_collision_probability", "collision_probability", max)):
+            v = c.get(src)
+            if v is None:
+                continue
+            agg[key] = v if agg[key] is None else fn(agg[key], v)
+
+    return {"params": p, "conflicts": conflicts, "aggregate": agg,
+            "measures_glossary": {
+                "TTC": "time-to-collision via rectangle conflict-point projection",
+                "PET": "post-encroachment time via zone-grid occupancy",
+                "DRAC": "deceleration rate to avoid crash = closing^2/(2*gap)",
+                "DeltaS": "max relative speed differential during conflict",
+                "MaxS": "max absolute speed during conflict",
+                "CSI": "time-weighted sum of 0.5*mu*DeltaS^2 * P(collision) * dt",
+            }}
+
+
+def _analyze_pair(traj_a: Dict, traj_b: Dict, p: Dict) -> List[Dict]:
+    dims_a = (traj_a.get("length", 0.0), traj_a.get("width", 0.0))
+    dims_b = (traj_b.get("length", 0.0), traj_b.get("width", 0.0))
+    mass_a = MASS_PROXY.get(traj_a.get("vehicle_type", "car"), 1.0)
+    mass_b = MASS_PROXY.get(traj_b.get("vehicle_type", "car"), 1.0)
+    mu = (mass_a * mass_b) / max(mass_a + mass_b, 1e-9)  # reduced mass proxy
+
+    by_t_b = {round(s["t"], 6): s for s in traj_b["states"]}
+    samples = []
+    for sa in traj_a["states"]:
+        sb = by_t_b.get(round(sa["t"], 6))
+        if sb is None:
+            continue
+        ttc = projected_ttc(sa, dims_a, sb, dims_b, p["look_ahead_s"], p["projection_dt_s"])
+        samples.append((sa, sb, ttc))
+
+    pet = zone_grid_pet(traj_a, traj_b, p["grid_cell_m"])
+
+    # Contiguous samples with TTC below threshold form one conflict event.
+    events: List[List[Tuple[Dict, Dict, float]]] = []
+    current: List[Tuple[Dict, Dict, float]] = []
+    for sa, sb, ttc in samples:
+        active = ttc is not None and ttc <= p["ttc_threshold_s"]
+        if active:
+            current.append((sa, sb, ttc))
+        elif current:
+            events.append(current)
+            current = []
+    if current:
+        events.append(current)
+
+    ts = sorted(s["t"] for s in traj_a["states"])
+    dt = (ts[1] - ts[0]) if len(ts) >= 2 else 0.1
+
+    conflicts = []
+    for ev in events:
+        delta_s = max_s = 0.0
+        max_drac_v: Optional[float] = None
+        min_ttc = None
+        max_p = 0.0
+        csi = 0.0
+        exposure = 0.0
+        for sa, sb, ttc in ev:
+            vax, vay = _velocity(sa)
+            vbx, vby = _velocity(sb)
+            ds = math.hypot(vax - vbx, vay - vby)
+            delta_s = max(delta_s, ds)
+            max_s = max(max_s, sa["speed"], sb["speed"])
+            d = drac(sa, dims_a, sb, dims_b)
+            if d is not None:
+                max_drac_v = d if max_drac_v is None else max(max_drac_v, d)
+            min_ttc = ttc if min_ttc is None else min(min_ttc, ttc)
+            prob = collision_probability(ttc, p["ttc_threshold_s"])
+            max_p = max(max_p, prob)
+            csi += 0.5 * mu * ds ** 2 * prob * dt
+            exposure += prob * dt
+        sa0, sb0, ttc0 = ev[0]
+        min_sample = min(ev, key=lambda e: e[2])
+        sa_m, sb_m, ttc_m = min_sample
+        vax, vay = _velocity(sa_m)
+        vbx, vby = _velocity(sb_m)
+        cp = ((sa_m["x"] + sb_m["x"]) / 2 + (vax + vbx) / 2 * ttc_m,
+              (sa_m["y"] + sb_m["y"]) / 2 + (vay + vby) / 2 * ttc_m)
+        conflicts.append({
+            "vehicle_a": traj_a["vehicle_id"],
+            "vehicle_b": traj_b["vehicle_id"],
+            "conflict_type": _conflict_type(sa_m["heading"], sb_m["heading"], p),
+            "t_start_s": round(sa0["t"], 3),
+            "t_end_s": round(ev[-1][0]["t"], 3),
+            "duration_s": round(ev[-1][0]["t"] - sa0["t"] + dt, 3),
+            "min_ttc_s": round(min_ttc, 4),
+            "pet_s": None if pet is None else round(pet, 4),
+            "max_drac_mps2": None if max_drac_v is None else round(max_drac_v, 4),
+            "delta_s_mps": round(delta_s, 4),
+            "max_s_mps": round(max_s, 4),
+            "collision_probability": round(max_p, 4),
+            "probability_exposure_s": round(exposure, 4),
+            "csi": round(csi, 6),
+            "mass_proxy_reduced": round(mu, 4),
+            "conflict_point": [round(cp[0], 2), round(cp[1], 2)],
+        })
+    return conflicts
+
+
+# ------------------------------------------------------------------ synthetic TRJ data
+
+
+def _states(xy_fn, speed_fn, heading: float, duration: float, dt: float) -> List[Dict]:
+    out = []
+    t = 0.0
+    while t <= duration + 1e-9:
+        x, y = xy_fn(t)
+        out.append({"t": round(t, 4), "x": round(x, 4), "y": round(y, 4),
+                    "speed": round(max(speed_fn(t), 0.0), 4), "heading": heading})
+        t += dt
+    return out
+
+
+def _braking_profile(v0: float, t_brake: float, decel: float):
+    """Position/speed along a 1-D line: constant v0, then braking at decel."""
+    t_stop = t_brake + v0 / decel
+
+    def pos(t: float) -> float:
+        if t <= t_brake:
+            return v0 * t
+        tb = min(t, t_stop) - t_brake
+        return v0 * t_brake + v0 * tb - 0.5 * decel * tb ** 2
+
+    def speed(t: float) -> float:
+        if t <= t_brake:
+            return v0
+        return max(0.0, v0 - decel * (t - t_brake))
+
+    return pos, speed
+
+
+def scenario_crossing(reaction_delay_s: float = 0.6, duration: float = 8.0,
+                      dt: float = 0.1, seed: int = 0) -> List[Dict]:
+    """Ego approaches a crossing pedestrian; brakes reaction_delay after t_detect=1s."""
+    rng = np.random.default_rng(seed)
+    v_ego = 12.5 + float(rng.uniform(-0.3, 0.3))
+    ped_v = 1.9 + float(rng.uniform(-0.1, 0.1))
+    pos, speed = _braking_profile(v_ego, 1.0 + reaction_delay_s, 5.0)
+    ego = {"vehicle_id": "ego", "vehicle_type": "car", "length": 4.5, "width": 1.9,
+           "states": _states(lambda t: (-42.0 + pos(t), 0.0), speed, 0.0, duration, dt)}
+    ped = {"vehicle_id": "ped-1", "vehicle_type": "pedestrian", "length": 0.6, "width": 0.6,
+           "states": _states(lambda t: (0.0, -6.5 + ped_v * t), lambda t: ped_v,
+                             math.pi / 2, duration, dt)}
+    return [ego, ped]
+
+
+def scenario_rear_end(reaction_delay_s: float = 0.6, duration: float = 8.0,
+                      dt: float = 0.1, seed: int = 0) -> List[Dict]:
+    """Lead vehicle brakes hard at t=1s; ego brakes reaction_delay later."""
+    rng = np.random.default_rng(seed + 100)
+    v = 15.0 + float(rng.uniform(-0.4, 0.4))
+    lead_pos, lead_speed = _braking_profile(v, 1.0, 6.0)
+    ego_pos, ego_speed = _braking_profile(v, 1.0 + reaction_delay_s, 5.5)
+    lead = {"vehicle_id": "lead", "vehicle_type": "car", "length": 4.5, "width": 1.9,
+            "states": _states(lambda t: (18.0 + lead_pos(t), 0.0), lead_speed, 0.0, duration, dt)}
+    ego = {"vehicle_id": "ego", "vehicle_type": "car", "length": 4.5, "width": 1.9,
+           "states": _states(lambda t: (0.0 + ego_pos(t), 0.0), ego_speed, 0.0, duration, dt)}
+    return [ego, lead]
+
+
+def scenario_lane_change(reaction_delay_s: float = 0.6, duration: float = 8.0,
+                         dt: float = 0.1, seed: int = 0) -> List[Dict]:
+    """Cyclist merges toward the ego lane at a shallow (lane-change) angle."""
+    rng = np.random.default_rng(seed + 200)
+    v_ego = 11.0 + float(rng.uniform(-0.3, 0.3))
+    v_cyc = 4.5
+    heading = math.radians(35.0)  # oblique merge toward the ego lane
+    pos, speed = _braking_profile(v_ego, 0.8 + reaction_delay_s, 4.5)
+    ego = {"vehicle_id": "ego", "vehicle_type": "car", "length": 4.5, "width": 1.9,
+           "states": _states(lambda t: (-14.0 + pos(t), 0.0), speed, 0.0, duration, dt)}
+    cyc = {"vehicle_id": "cyclist-1", "vehicle_type": "cyclist", "length": 1.8, "width": 0.6,
+           "states": _states(lambda t: (2.0 + v_cyc * math.cos(heading) * t,
+                                        -3.0 + v_cyc * math.sin(heading) * t),
+                             lambda t: v_cyc, heading, duration, dt)}
+    return [ego, cyc]
+
+
+_SCENARIOS = {
+    "crossing": scenario_crossing,
+    "rear_end": scenario_rear_end,
+    "lane_change": scenario_lane_change,
+}
+
+
+def generate_trajectories(seed: int = 0, scenario: str = "mixed",
+                          reaction_delay_s: float = 0.6,
+                          duration: float = 8.0, dt: float = 0.1) -> List[Dict]:
+    """Deterministic synthetic TRJ-like trajectory data to exercise the analyzer."""
+    if scenario in _SCENARIOS:
+        return _SCENARIOS[scenario](reaction_delay_s, duration, dt, seed)
+    if scenario != "mixed":
+        raise ValueError(f"unknown scenario {scenario!r}; expected one of "
+                         f"{sorted(_SCENARIOS)} or 'mixed'")
+    out: List[Dict] = []
+    for name, fn in _SCENARIOS.items():
+        for traj in fn(reaction_delay_s, duration, dt, seed):
+            traj = dict(traj)
+            traj["vehicle_id"] = f"{name}:{traj['vehicle_id']}"
+            out.append(traj)
+    return out
+
+
+# ------------------------------------------------------------------ run-level CSI
+
+
+def reaction_delay_for_model(model_version: str, overrides: Optional[Dict] = None) -> Dict:
+    """Deterministic mapping: perception skill profile -> downstream reaction delay.
+
+    SIMULATED: a weaker perception stack (higher miss penalties) is modeled as a
+    longer effective detection-to-brake delay, which the surrogate-safety
+    scenarios convert into more severe conflicts. Marked simulated; the mapping
+    is monotonic and seeded by the same model_params used by megaeval runs.
+    """
+    from sensorflow.megaeval.runs import model_params
+    params = model_params(model_version, overrides or {})
+    perception_score = params["base_detect"] - (
+        params["night_penalty"] + params["rain_penalty"]
+        + params["occlusion_penalty"] + params["vru_penalty"])
+    deficit = float(np.clip(1.0 - perception_score, 0.0, 1.0))
+    delay = 0.45 + 1.6 * deficit
+    return {"model_version": model_version, "perception_score": round(perception_score, 4),
+            "perception_deficit": round(deficit, 4),
+            "reaction_delay_s": round(delay, 4), "simulated": True}
+
+
+def csi_for_run(run, seeds: Tuple[int, ...] = (0, 1, 2), force: bool = False) -> Dict:
+    """Aggregate CSI for a megaeval evaluation run (cached under runs/safety/ssam/).
+
+    Runs the three surrogate-safety scenarios (crossing / rear-end / lane-change)
+    with the model-conditioned reaction delay and sums CSI across all conflicts.
+    """
+    cached = None if force else read_json("ssam", f"{run.run_id}.json")
+    if cached is not None:
+        return cached
+
+    reaction = reaction_delay_for_model(run.model_version, run.overrides)
+    scenario_rows = []
+    total_csi = 0.0
+    total_conflicts = 0
+    min_ttc = None
+    for name in _SCENARIOS:
+        for seed in seeds:
+            trajs = generate_trajectories(seed=seed, scenario=name,
+                                          reaction_delay_s=reaction["reaction_delay_s"])
+            res = analyze_trajectories(trajs)
+            agg = res["aggregate"]
+            total_csi += agg["aggregate_csi"]
+            total_conflicts += agg["num_conflicts"]
+            if agg["min_ttc_s"] is not None:
+                min_ttc = agg["min_ttc_s"] if min_ttc is None else min(min_ttc, agg["min_ttc_s"])
+            scenario_rows.append({"scenario": name, "seed": seed,
+                                  "num_conflicts": agg["num_conflicts"],
+                                  "min_ttc_s": agg["min_ttc_s"],
+                                  "max_drac_mps2": agg["max_drac_mps2"],
+                                  "aggregate_csi": agg["aggregate_csi"]})
+
+    summary = {
+        "run_id": run.run_id,
+        "model_version": run.model_version,
+        "reaction_model": reaction,
+        "scenarios": scenario_rows,
+        "total_conflicts": total_conflicts,
+        "min_ttc_s": min_ttc,
+        "aggregate_csi": round(total_csi, 6),
+        "simulated": True,
+        "method": "deterministic surrogate-safety scenario suite; CSI = "
+                  "time-weighted kinetic-energy proxy x collision probability",
+    }
+    write_json(summary, "ssam", f"{run.run_id}.json")
+    return summary

sensorflow/safety/store.py

@@ -0,0 +1,60 @@
+"""Filesystem layout + JSON helpers for the safety layer.
+
+All safety artifacts persist under runs/safety/ (relocatable for tests via
+set_safety_root, mirroring megaeval's set_mega_root pattern):
+
+    runs/safety/
+        gate_policy.json           configurable release-gate thresholds
+        odd/{run_id}.json          synthetic gap-fill supplements per evaluation run
+        ssam/{run_id}.json         cached surrogate-safety (CSI) summaries per run
+        calibration/status.json    latest multi-sensor calibration validation
+        discrepancy/{dataset}.json online-vs-offline mining results
+        evidence/{run_id}.json/.md safety evidence packages
+        scenario_db.json           curated scenario database
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+from typing import Any, Optional
+
+SAFETY_ROOT = os.path.join("runs", "safety")
+
+_LOCK = threading.RLock()
+
+
+def set_safety_root(path: str) -> None:
+    """Test hook: relocate all safety-layer storage."""
+    global SAFETY_ROOT
+    SAFETY_ROOT = str(path)
+
+
+def safety_path(*parts: str) -> str:
+    """Absolute-ish path under the safety root; parent dirs are created."""
+    path = os.path.join(SAFETY_ROOT, *parts)
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    return path
+
+
+def read_json(*parts: str) -> Optional[Any]:
+    path = os.path.join(SAFETY_ROOT, *parts)
+    if not os.path.exists(path):
+        return None
+    with _LOCK:
+        try:
+            with open(path) as f:
+                return json.load(f)
+        except Exception:
+            return None
+
+
+def write_json(payload: Any, *parts: str) -> str:
+    path = safety_path(*parts)
+    with _LOCK:
+        tmp = path + ".tmp"
+        with open(tmp, "w") as f:
+            json.dump(payload, f, indent=2)
+        os.replace(tmp, path)
+    return path

sensorflow/seqeval/DESIGN.md

@@ -0,0 +1,449 @@
+# seqeval — Efficient Sequential Regression Detection: Technical Design
+
+**Goal.** Detect small model regressions (~2 percentage points on specific
+classes/conditions) on every model update WITHOUT evaluating the full corpus,
+with statistical validity under (a) sequential looks, (b) clustered
+(non-independent) frames, (c) stratified non-proportional sampling, and
+(d) many simultaneous hypotheses.
+
+This document records the design *and the judgment calls*: which candidate
+techniques were adopted, which were rejected, and why. The reference
+implementation lives in this package and is exercised by `tests/test_seqeval/`.
+
+---
+
+## 1. Problem formulation
+
+For each metric m (recall is the driver metric in v1) and each population
+stratum h, let Δ_h = m_candidate(h) − m_baseline(h) measured on the same data.
+
+With a **practical-significance margin** δ (default 0.005 = 0.5pp):
+
+```
+H0 (no material regression):  Δ_h ≥ −δ
+H1 (material regression):     Δ_h < −δ
+```
+
+plus the separate equivalence-style claim used for PASS: reject `Δ_h ≤ −δ`
+in favor of `Δ_h > −δ`.
+
+**Absolute vs relative deltas.** The default driver is **absolute percentage
+points**, for three reasons:
+
+1. Safety cost is absolute: a 2pp recall drop means the same number of extra
+   missed pedestrians whether baseline recall was 0.98 or 0.80. Relative
+   deltas overstate changes on strong baselines (0.98→0.96 is "only −2%
+   relative") and understate them on weak ones.
+2. Absolute deltas keep Bernoulli variance algebra and margin semantics simple
+   and uniform across strata; relative margins make the null hypothesis depend
+   on the (estimated) baseline value, contaminating the test with baseline
+   estimation error.
+3. Absolute margins are what promotion policies elsewhere in this platform
+   already use (`megaeval.analysis.DEFAULT_PROMOTION_POLICY` is expressed in
+   absolute drops), so the gate composes with existing policy language.
+
+Relative deltas are still *reported* in every evidence record (`rel_delta`)
+because they are useful for human interpretation, especially for rare classes
+with weak baselines. They are never the decision variable.
+
+**Practical vs statistical significance.** With ~320k objects, a 0.05pp drop
+can be "statistically significant" yet operationally meaningless. Hence the
+margin δ: nothing is declared REGRESSION unless the evidence shows the drop
+exceeds δ, and nothing is declared PASS unless the evidence shows any drop is
+smaller than δ. Statistical significance against a point null of exactly zero
+is deliberately not used anywhere.
+
+## 2. The statistical unit: clusters, not frames
+
+Frames/objects are not independent: objects in one container (scene/track)
+share weather, lighting, scenario, scene-level hardness and — critically —
+correlated model failures (one bad scene produces many misses). Treating
+objects as iid understates variance by the design effect
+
+```
+DEFF = 1 + (m̄ − 1) · ICC        (m̄ = size-weighted mean cluster size)
+```
+
+and every "significant" result computed that way is anti-conservative.
+
+Adopted design:
+
+* **Inference unit = cluster**: all sampled objects of a stratum in one
+  container. Per-cluster mean paired differences (bounded in [−1, 1]) are the
+  observations fed to the confidence sequences and e-processes. This is valid
+  under *arbitrary* within-cluster correlation — no correlation model is
+  assumed — and whole clusters are sampled so the unit is never split
+  (`planner.py` selects container groups, not objects).
+* **Diagnostics, not corrections**: ICC (one-way ANOVA estimator), DEFF and
+  n_eff = n/DEFF are computed per node (`units.py`) and attached to every
+  evidence record. They are reporting artifacts; validity does not depend on
+  them because clustering is handled structurally.
+* An interesting empirical note the machinery surfaces: pairing itself removes
+  most cluster correlation (the shared scene effect hits both models and
+  cancels in d_i), so ICC on paired differences is small (~0.07 in the worked
+  example, DEFF ≈ 1.2) even when ICC on raw outcomes is large. We keep the
+  cluster unit anyway — the residual correlation is real and free to handle.
+* **Clustered bootstrap** (resampling clusters, not objects) is the prescribed
+  fallback for metrics that don't decompose into per-object outcomes (§3). It
+  is not needed for the v1 recall driver.
+
+Rejected: variance-inflation-only corrections (divide n by an assumed DEFF≈2
+and pretend units are iid). Rejected because ICC varies by stratum and model
+pair; a fixed inflation is either wasteful or invalid, and it composes poorly
+with anytime-valid methods whose guarantees are per-observation.
+
+## 3. Paired evaluation as the core design
+
+Both models are evaluated on the **same sampled units** (`paired.py`), and all
+inference is on the per-unit difference d_i = c_i − b_i ∈ {−1, 0, +1}.
+
+Why this is the single biggest efficiency lever: for two independent samples,
+Var(m̂_c − m̂_b) ��� 2·p(1−p)/n (~0.18/n at p=0.9). For paired evaluation,
+Var(d) = ν − Δ², where ν = P(models disagree) — the discordance rate. Adjacent
+model versions agree on almost everything (measured ν ≈ 0.03–0.06 in the
+harness), so pairing cuts required samples by roughly 3–10×. The paired
+design also removes the between-unit difficulty variance *and* the shared
+scene effects (§2).
+
+**McNemar correspondence.** For binary per-object outcomes the information in
+the paired design is exactly the discordant counts: n01 (baseline right,
+candidate wrong) vs n10 (the reverse) — McNemar's setup. We do not use the
+classical McNemar *test* (it is fixed-n); instead the same discordant-pair
+information drives (a) the sequential e-process on d_i and (b) the
+Beta-Binomial posterior of §14, which is literally a Bayesian McNemar. The
+counts n01/n10 are reported in attribution because they are the most
+interpretable summary of "what changed".
+
+**Which metrics get which treatment:**
+
+| Metric | Per-object outcome? | Sequential treatment |
+|---|---|---|
+| recall (per-GT-object TP/FN) | yes, Bernoulli | confidence sequence / e-process on paired d_i ✔ (v1 driver) |
+| precision (per-prediction) | yes, but the unit set differs per model (each model's own predictions); pair at the cluster level | CS on cluster-level paired precision differences |
+| safety_recall, per-stratum recall | yes | same as recall ✔ |
+| mean IoU (matched pairs) | bounded continuous per object | CS on paired IoU differences (empirical-Bernstein handles [0,1] naturally) |
+| AP / mAP / AUC | **no** — rank statistics over the whole set; not a mean of per-object terms | **paired clustered bootstrap** at checkpoints (resample clusters, recompute both models' AP on the same resample, CI on the difference). No anytime-valid guarantee: schedule at *pre-registered* checkpoints (end of each budget stage) with alpha-spending across the (few, fixed) looks |
+
+This split is a deliberate honesty boundary: Bernoulli-style anytime-valid
+treatment is used exactly where a metric is a mean of bounded per-unit terms;
+composite rank metrics get bootstrap at a small number of pre-committed looks
+rather than pretending they support continuous monitoring.
+
+## 4. Sampling design: hybrid Neyman + risk floors + safety minimums
+
+Candidates considered:
+
+* **Proportional allocation** — matches the population, but starves exactly
+  the strata we care about (pedestrian|night is ~5% of objects; a proportional
+  10k sample gives ~500 units, hopeless for 2pp detection). Rejected as the
+  primary rule.
+* **Equal allocation** — maximal per-stratum power for fixed budget, but
+  wastes budget on huge easy strata and makes population-level estimates
+  needlessly noisy. Rejected.
+* **Risk-weighted only** (as in `megaeval.sampling`) — right instinct, no
+  optimality story; can be arbitrarily far from variance-optimal. Rejected as
+  the *sole* rule, kept as a multiplier.
+* **Neyman allocation** n_h ∝ N_h·σ_h — variance-optimal for the *population*
+  estimate, but per-stratum guarantees are incidental, and σ_h must not come
+  from candidate outcomes (bias firewall, §13). Adopted as the baseline using
+  a candidate-independent PRIOR variance profile (documented rates by
+  class/condition; a pilot baseline-only estimate would also qualify).
+
+**Adopted hybrid** (`planner.py`), in order: Neyman shares from prior σ_h →
+multiplicative risk weights (VRU ×1.5, night ×1.3) → mandatory minimum per
+stratum (200) → pre-registered safety-critical strata get a floor sized for
+the target MDE (2500 by default, §10) → truncate at N_h, select whole
+clusters in a seeded shuffled order.
+
+**Unbiasedness under oversampling.** Every sampled object carries the
+Horvitz–Thompson weight w_h = N_h/n_h. Population and class-level estimators
+are weight-corrected (weighted cluster means feed the overall/class/difficulty
+nodes), so the classic stratified identity `E[Σ_h W_h p̂_h] = p` holds and
+oversampling safety strata does not bias headline estimates. Per-stratum
+(leaf) inference is self-weighting and needs no correction.
+
+## 5. Multi-stage pipeline
+
+```
+sanity (≈500 units)  →  stratified screening (~15% of plan)
+    →  sequential confirmation (batched to 100% of plan, anytime-valid)
+    →  targeted escalation (reserve clusters, suspect/safety strata only)
+```
+
+* **Sanity**: fingerprints resolved, frozen plan hashed, smoke batch checks
+  for degenerate outputs (rate ∉ (0.2, 1], discordance ≥ 0.5 ⇒ abort — a
+  broken export should fail in seconds, not consume the budget).
+* **Screening** flags "suspect" strata early (e-value ≥ 1.5 — soft evidence,
+  *never* a decision) to prioritize escalation later. Screening consumes the
+  same frozen plan; it does not select units based on outcomes (§13).
+* **Sequential confirmation** processes the remaining plan in batches; all
+  decisions (REGRESSION / PASS) can fire at any batch boundary because the
+  underlying processes are anytime-valid.
+* **Escalation** pulls from each stratum's pre-frozen reserve (shuffled at
+  plan time) for strata still undecided that are suspect or safety-primary,
+  up to a hard cap. Escalation to *full* evaluation of a stratum is the
+  terminal step of the same mechanism (the reserve is the rest of the
+  stratum).
+
+## 6. Sequential methodology: anytime-valid confidence sequences, not CUSUM
+
+**Why repeated fixed-sample CIs are wrong.** A 95% CI computed at n has 5%
+miscoverage *at that single n*. Recompute after every batch and stop on the
+first exclusion of the null and the type-I error inflates monotonically —
+by the law of the iterated logarithm the statistic crosses any fixed-n
+boundary infinitely often under the null, so "peek until significant" has
+error probability → 1. Evaluation gates that check after every batch are
+exactly this setting.
+
+**Adopted: confidence sequences / e-processes** (`sequential.py`):
+
+* Empirical-Bernstein confidence sequence (Waudby-Smith & Ramdas 2023,
+  predictable plug-in): a CS valid *simultaneously over all n*, so the gate
+  may look after every batch and stop whenever it likes. Empirical-Bernstein
+  rather than Hoeffding because paired differences have tiny variance
+  (ν ≪ 1/4) and EB width scales with the realized variance; a Hoeffding CS
+  would be ~3–4× wider here (Hoeffding rejected for power, kept as a mental
+  baseline).
+* One-sided **betting e-processes** for the two directed nulls. The wealth
+  process K_t = Π(1 − λ_t(X_t − m)) is a nonnegative supermartingale under
+  H0: μ ≥ m for any predictable λ_t in the admissible range; Ville's
+  inequality gives P(∃t: K_t ≥ 1/α) ≤ α. Bet sizing = max(empirical-Bernstein
+  recipe, aGRAPA/approximate-Kelly λ ≈ drift/(σ̂² + drift²)), clipped —
+  sizing affects power only, never validity. E-values (not just CIs) are kept
+  because they compose: they feed e-BH in the hierarchy (§8) and multiply
+  across independent looks/runs if evidence is ever pooled.
+* Batch-incremental updates, sticky decisions, per-batch trajectory recording
+  (`log e vs n` with the 1/α boundary) for the dashboard.
+
+**Where CUSUM legitimately fits — and why not here.** CUSUM/Page tests are
+designed for *change-point detection in an ongoing stream*: an unknown time
+at which a monitored process shifts, with guarantees phrased as average run
+length (ARL) to false alarm and detection delay. That is the right tool for
+**production drift monitoring** (a deployed model's live metric degrading at
+an unknown time — a reasonable future addition next to
+`sensorflow/evaluation/regression.py`). It is the wrong tool for **gated A/B
+evaluation**, where there is no change-point: the candidate is a fixed,
+already-different model, the hypothesis is about a fixed mean difference, ARL
+semantics don't map to per-release type-I error, and CUSUM provides neither
+confidence intervals for effect size nor e-values for hierarchical multiple
+testing. CUSUM is therefore explicitly rejected for the gate.
+(Group-sequential designs — O'Brien-Fleming etc. — were also considered:
+valid, but they require pre-committing to a fixed number of looks and a
+maximum n, which fights the budget-staged, stop-any-time controller; the
+anytime-valid framework strictly dominates for this use.)
+
+## 7. Mandatory three-outcome decisions
+
+Non-detection is NEVER equivalence. Each node ends in exactly one of:
+
+* **REGRESSION** — the regression e-process crossed 1/α_allocated: confirmed
+  drop beyond the margin. Gate: block.
+* **PASS** — the equivalence e-process crossed 1/α_pass: confirmed that any
+  drop is smaller than the margin (this is a one-sided equivalence /
+  non-inferiority claim, the sequential analogue of a TOST bound). Gate:
+  allow, but only when the overall node AND every pre-registered safety
+  primary individually PASS.
+* **INSUFFICIENT_EVIDENCE** — budget exhausted first. Gate: expand within
+  budget if possible, else report with explicit language: *"not proven
+  equivalent — do not treat as a pass."* (`controller.INSUFFICIENT_LANGUAGE`
+  is exactly that sentence; the tiny-budget test asserts it.)
+
+## 8. Hierarchical multiple testing
+
+Testing ~28 nodes (overall, 6 classes, 18 class×condition strata, 3 difficulty
+bands) at α each would give a family error of ~1−0.95²⁸ ≈ 76%. Naive
+Bonferroni over everything destroys power for the one node that matters.
+Adopted structure (`hierarchy.py`):
+
+```
+Level 1  overall                        α·0.30   single test
+Safety   pre-registered primaries       α·0.30   split equally, tested individually
+Level 2  class                          α·0.15   e-BH within level
+Level 3  class × condition strata       α·0.15   e-BH within level (non-safety)
+Level 4  difficulty bands               α·0.10   e-BH within level
+```
+
+Budgets sum to α, so the union bound gives family-wise control across levels;
+within a level, **e-BH** (BH on e-values) controls FDR *under arbitrary
+dependence* — mandatory here because levels re-aggregate the same units and
+strata share scenes. e-BH composes with optional stopping because an
+e-process value at any stopping time is a valid e-value (this is the reason
+the implementation carries e-values and not just CSs).
+
+**Gatekeeping and the masking trap.** Classic top-down gatekeeping ("test
+children only where the parent rejected") was **rejected** for the regression
+direction: a 2pp pedestrian-night drop inside a globally *improved* model is
+invisible at level 1, and a gate that only descends on level-1 rejection would
+never see it. Instead every level has its own reserved budget, and
+**safety-critical strata are pre-registered primary hypotheses** with
+dedicated alpha, tested individually and exempt from e-BH masking by
+better-behaved siblings. The power test in the suite is exactly this case:
+overall metric up ~0.3pp, pedestrian-night down 2pp — and the stratum node
+must fire. Alpha propagation is top-down only in the *reporting* sense
+(attribution descends into difficulty bands of regressed strata); decisions
+never require a parent rejection.
+
+## 9. Rare-class policy
+
+* **Minimum counts**: every non-empty stratum gets ≥ 200 sampled objects
+  (or all of N_h if smaller) — below that, even 10pp effects are undetectable
+  and the honest answer is INSUFFICIENT_EVIDENCE, which the three-outcome
+  logic produces naturally.
+* **Oversampling with weight correction**: rare + risky strata get risk
+  multipliers and floors (§4); HT weights keep population estimates unbiased.
+* **Per-class thresholds**: the margin δ and alpha allocation are policy
+  fields; safety primaries effectively run at tighter per-node alpha with far
+  larger minimum samples. (A per-stratum δ_h map is a straightforward policy
+  extension; v1 keeps a single δ for legibility.)
+* **Escalation**: rare strata that end INSUFFICIENT but suspect consume their
+  reserve up to the whole stratum. If a stratum is exhausted and still
+  undecided, the ledger says so explicitly (stopping_reason
+  `escalation_exhausted`) — the decision is a visible budget/detectability
+  statement, not silence.
+
+## 10. Sample sizes and power (with the design effect)
+
+Fixed-n planning numbers for a Bernoulli recall metric at baseline p₀ = 0.9,
+one-sided α = 0.05, power 0.9 ((z₀.₉₅+z₀.₉)² ≈ 8.57). Paired variance uses
+Var(d) ≈ ν − Δ² with measured-typical discordance ν ≈ 0.04 + Δ:
+
+| detectable Δ | unpaired, per model arm | paired, iid units | paired, DEFF = 2 | paired, DEFF = 2, anytime (~×1.35) |
+|---:|---:|---:|---:|---:|
+| 1pp  | 16,100 | 4,280 | 8,550 | ~11,500 |
+| 2pp  |  4,150 | 1,280 | 2,550 |  ~3,450 |
+| 5pp  |    745 |   300 |   600 |    ~810 |
+| 10pp |    214 |   111 |   223 |    ~300 |
+
+Notes: (i) pairing beats doubling the corpus — the unpaired column is per
+*arm*; (ii) the anytime-valid overhead (~30–40% at these scales, shrinking
+with the e-process's adaptive bets) buys the right to stop at any time, which
+in expectation *saves* samples whenever the effect is larger than the MDE
+(the 10pp test stops ~15× before budget); (iii) measured DEFF on paired
+differences was ≈1.2, not 2 — the table's DEFF=2 column is the conservative
+planning figure requested, the ledger reports the realized one. The default
+safety floor (2,500) is the 2pp row with the realized design effect plus
+head-room.
+
+## 11. Compute-budget staging
+
+Default policy on a 320k corpus: plan ≈ 25k objects, staged
+`[15%, 40%, 100%]` of plan (≈ 3.8k → 10k → 25k) + escalation reserve, i.e. a
+10K→50K→250K→full ladder scaled to plan size. Advancement rules: advance
+while any node is undecided and budget remains; stop immediately on any
+confirmed REGRESSION (sticky, anytime-valid) or when overall + all safety
+primaries PASS with nothing undecided; escalate only suspect/safety strata;
+terminal state otherwise INSUFFICIENT_EVIDENCE with per-stratum sample
+accounting in the ledger.
+
+## 12. Prediction caching
+
+Baseline predictions are computed once per (dataset fingerprint, model
+fingerprint) and reused across every candidate update
+(`paired.PredictionCache`, disk + memory). Fingerprints cover everything
+predictions depend on — dataset identity/seed/size, model version + effects
+config — so **invalidation is by construction**: any change produces a new
+key; nothing stale can ever be served. A compute counter proves in tests that
+the second candidate evaluation performs zero baseline recomputation. In a
+real deployment the fingerprint inputs become dataset content hash + model
+checkpoint digest + inference config (thresholds, NMS, quantization), and the
+cache stores per-unit predictions rather than a success vector.
+
+## 13. Sampling-bias prevention
+
+* The plan is built by a function that **cannot see candidate outcomes** (no
+  such parameter exists), from population structure + a prior variance
+  profile only, then persisted with seed, config and a content hash over the
+  selected and reserve unit ids. The hash is recorded in every run's lineage;
+  two candidates with opposite outcomes provably receive byte-identical plans
+  (asserted in tests).
+* Escalation draws from a reserve whose order was frozen at plan time, so
+  even adaptive *budget* decisions never become adaptive *selection*.
+* **If outcome-adaptive sampling is ever added** (e.g. active sampling of
+  high-disagreement regions), estimates must be corrected with
+  inverse-propensity weights w_i = 1/π_i under recorded, everywhere-positive
+  selection probabilities π_i, and CS validity must be re-established for the
+  weighted observations (bounded w_i·d_i, e.g. via truncated propensities).
+  Without that correction, adaptive selection biases Δ̂ toward whatever the
+  selector chases. v1 deliberately does not do adaptive selection.
+
+## 14. Bayesian complement
+
+Alongside every frequentist decision, `sequential.py` computes a Beta-Binomial
+posterior P(Δ < −δ | data) from the discordant pairs (θ = P(regression pair |
+discordant) with Beta(1,1) prior; Δ = ν(1−2θ)). Operational comparison:
+
+* *For* the posterior: a single monotone number ("99.99% probable this is a
+  real regression") that program managers read correctly without training;
+  natural for a decision-theoretic cost tradeoff.
+* *Against* (as the gate): posteriors have no frequentist stopping guarantee
+  under continuous monitoring without committing to a prior + loss and
+  accepting their audit burden; e-values give the same "accumulating
+  evidence" narrative *with* an error guarantee any auditor can check
+  (e ≥ 1/α), and they compose across strata (e-BH) and across looks by
+  multiplication.
+
+Verdict: **e-values gate, posterior explains.** Both are in the ledger.
+
+## 15. Failure modes and mitigations
+
+| Failure mode | Mitigation in this design |
+|---|---|
+| Peeking inflates type-I | everything anytime-valid; fixed-n intervals never gate (§6) |
+| Correlated frames overstate n | cluster inference units; DEFF/n_eff in every record (§2) |
+| Masking: overall improves, stratum regresses | reserved per-level alpha + pre-registered safety primaries (§8) |
+| Multiplicity across strata | hierarchical budgets + e-BH within level (§8) |
+| "No signal" read as "safe" | three-outcome logic; INSUFFICIENT carries "not proven equivalent" (§7) |
+| Outcome-dependent sampling bias | frozen hashed plans, frozen reserves, no adaptive selection (§13) |
+| Rare strata silently undecidable | floors, oversampling+weights, escalation, explicit exhaustion reason (§9) |
+| Stale baseline predictions | fingerprint-keyed cache, invalidation by construction (§12) |
+| Broken candidate export burns budget | sanity smoke stage aborts in ~500 units (§5) |
+| Trivial-magnitude "significance" | practical margin δ in both null hypotheses (§1) |
+| Irreproducible decisions | seeds + fingerprints + plan hash + config in lineage; end-to-end determinism asserted in tests |
+| Metric doesn't decompose per object (mAP) | pre-registered checkpoint paired clustered bootstrap, not fake sequential (§3) |
+
+## 16. Final recommendation (defaults shipped in `controller.DEFAULT_POLICY`)
+
+Paired, cluster-unit, stratified evaluation against a frozen hybrid
+Neyman/risk-floor plan; empirical-Bernstein confidence sequences + betting
+e-processes with a 0.5pp practical margin; three-outcome decisions; hierarchical
+alpha budget (30% overall / 30% safety primaries / 15% class / 15% stratum /
+10% difficulty) with e-BH within levels; staged budget 15%→40%→100% of a ~25k
+plan with targeted escalation; fingerprint-keyed baseline cache; Beta-Binomial
+posterior as explanatory complement. CUSUM reserved for future production
+drift monitoring, not release gating.
+
+## 17. Worked example: planted 2pp pedestrian-night regression (real run)
+
+Setup: population `320k` objects / 26,666 containers (megaeval synthetic,
+seed 42). Baseline `model-v41`. Candidate `model-v42` with planted effects
+{pedestrian|night: −2pp, global: +0.4pp} — the masking case. Default policy
+(δ = 0.5pp, α = 0.05, plan target 24k). Ground truth measured on the full
+corpus afterwards: overall Δ = **+0.28pp** (candidate looks *better*
+headline-wise), pedestrian|night Δ = **−1.63pp** realized (N = 17,561,
+baseline recall 0.801).
+
+Run (deterministic, reproducible from lineage):
+
+* Plan: 25,699 objects frozen across 18 strata; pedestrian|night allocated
+  2,501 (floor-driven; weight w = 7.02) + 15,060-object reserve.
+* Decision: **REGRESSION** at 26,705 objects total = **8.4% of the corpus**
+  (~12× cheaper than a full paired evaluation), stopping reason
+  `regression_confirmed`, during escalation of the suspect safety primary.
+* Pedestrian|night node at stop: n = 3,507 objects in 1,195 clusters
+  (ICC 0.065, DEFF 1.20, n_eff 2,923); Δ̂ = −1.94pp,
+  95% CS [−3.97pp, −0.48pp]; discordant pairs n01 = 110 vs n10 = 42;
+  log e-value **7.50** vs boundary log(1/α_safety) = **4.89**
+  (e ≈ 1,808 ≥ 133); Beta-Binomial P(Δ < −δ | data) = 0.99999.
+* Evidence trajectory (n, log e): (129, 1.5) → (1002, 1.6) → (2501, 2.8) →
+  (3006, 4.9) → (3507, 7.5) — the dashboard plots exactly this against the
+  4.89 boundary.
+* Overall node: Δ̂ = +0.03pp, CS [−0.51pp, +0.38pp] — correctly *not*
+  regressed and *not yet* provably equivalent; the hierarchy is what surfaced
+  the stratum despite the healthy headline.
+* Ledger: 28 node records, each with the full §15-proof field set; lineage
+  carries plan hash, fingerprints, seed and the entire statistical config.
+
+The same engine with a null candidate (no planted effect) across 24 Monte
+Carlo repetitions produced **zero** false REGRESSION verdicts (α = 0.05), and
+with a planted 10pp effect stopped at ~2.8k samples — 11% of plan, <1% of the
+corpus (both asserted in `tests/test_seqeval/`).

sensorflow/seqeval/__init__.py

@@ -0,0 +1,24 @@
+"""seqeval — budgeted, anytime-valid sequential regression detection.
+
+Detects small (~2pp) per-stratum model regressions without evaluating the full
+corpus, with validity under sequential looks, clustering, stratification and
+multiple comparisons. See DESIGN.md in this package for the full statistical
+design and its justifications.
+
+Module map:
+    units.py       statistical units: clusters, ICC, design effect
+    planner.py     frozen stratified sampling plans (Neyman + risk floors)
+    paired.py      paired harness + fingerprint-keyed prediction cache
+    sequential.py  empirical-Bernstein confidence sequences, e-processes,
+                   three-outcome decisions, Beta-Binomial complement
+    hierarchy.py   hierarchical gatekeeping with e-BH within levels
+    attribution.py the regression map
+    ledger.py      machine-readable evidence records + lineage
+    controller.py  sanity -> screening -> sequential -> escalation state machine
+    api.py         REST surface under /api/seqeval
+
+Entry point for the safety Regression Gate:
+    from sensorflow.seqeval import evaluate_regression
+"""
+
+from sensorflow.seqeval.controller import evaluate_regression  # noqa: F401

sensorflow/seqeval/api.py

@@ -0,0 +1,125 @@
+"""FastAPI router for the sequential regression-detection engine.
+
+Endpoints (prefix /api/seqeval):
+    POST /runs                start a sequential regression evaluation
+    GET  /runs                list runs
+    GET  /runs/{id}           state: stage, samples vs budget, per-node
+                              confidence sequences, decisions, and the
+                              sequential-evidence trajectories (points +
+                              stopping boundaries) for the dashboard chart
+    GET  /runs/{id}/evidence  machine-readable evidence ledger + lineage
+    GET  /runs/{id}/attribution  the regression map
+    GET  /policy              default statistical policy (documented)
+"""
+
+from __future__ import annotations
+
+from typing import Dict, Optional
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel, Field
+
+from sensorflow.seqeval import hierarchy as hierarchy_mod
+from sensorflow.seqeval import ledger as ledger_mod
+from sensorflow.seqeval.controller import (DEFAULT_POLICY, get_seqeval_store,
+                                           merged_policy)
+
+router = APIRouter(prefix="/api/seqeval")
+
+
+class ModelSpec(BaseModel):
+    model_version: str
+    effects: Dict[str, float] = Field(
+        default_factory=dict,
+        description="Per-stratum detection deltas, e.g. {'pedestrian|night': -0.02};"
+                    " '__global__' applies everywhere (synthetic model harness).")
+
+
+class StartRunRequest(BaseModel):
+    population_id: str
+    baseline: ModelSpec
+    candidate: ModelSpec
+    policy: Dict = Field(default_factory=dict)
+    seed: Optional[int] = None
+    sync: bool = Field(True, description="Run to completion before returning "
+                                         "(seconds); false = background thread.")
+
+
+@router.post("/runs")
+def start_run(req: StartRunRequest):
+    store = get_seqeval_store()
+    try:
+        run = store.create_run(req.population_id, req.baseline.model_dump(),
+                               req.candidate.model_dump(),
+                               policy=req.policy or None, seed=req.seed)
+    except KeyError as e:
+        raise HTTPException(404, str(e))
+    if req.sync:
+        store.execute_sync(run)
+    else:
+        store.start_async(run)
+    return run.to_dict(include_trajectories=False)
+
+
+@router.get("/runs")
+def list_runs():
+    return {"runs": get_seqeval_store().list_states()}
+
+
+def _state_or_404(run_id: str, include_trajectories: bool = True) -> Dict:
+    state = get_seqeval_store().get_state(run_id, include_trajectories=include_trajectories)
+    if state is None:
+        raise HTTPException(404, f"Unknown seqeval run {run_id}")
+    return state
+
+
+@router.get("/runs/{run_id}")
+def get_run(run_id: str):
+    return _state_or_404(run_id)
+
+
+@router.get("/runs/{run_id}/evidence")
+def get_evidence(run_id: str):
+    _state_or_404(run_id, include_trajectories=False)
+    ledger = ledger_mod.EvidenceLedger(run_id)
+    return {"run_id": run_id, "records": ledger.records(),
+            "lineage": ledger.lineage(),
+            "required_fields": ledger_mod.REQUIRED_FIELDS}
+
+
+@router.get("/runs/{run_id}/attribution")
+def get_attribution(run_id: str):
+    state = _state_or_404(run_id, include_trajectories=False)
+    store = get_seqeval_store()
+    run = store.runs.get(run_id)
+    if run is not None and run.attribution is not None:
+        att = run.attribution
+    else:
+        att = state.get("attribution")
+    if att is None:
+        raise HTTPException(409, f"Run {run_id} has no attribution yet "
+                                 f"(status={state.get('status')})")
+    return {"run_id": run_id, "decision": state.get("decision"),
+            "gate": state.get("gate"), **att}
+
+
+@router.get("/policy")
+def get_policy():
+    return {
+        "default_policy": merged_policy(),
+        "decision_semantics": {
+            "REGRESSION": "block: anytime-valid evidence of a drop beyond the "
+                          "practical margin in at least one tested node",
+            "PASS": "allow: overall + all pre-registered safety primaries proven "
+                    "within the margin (equivalence-style claim)",
+            "INSUFFICIENT_EVIDENCE": "expand budget or report: NOT proven "
+                                     "equivalent; never treated as a pass",
+        },
+        "test_method": ledger_mod.TEST_METHOD,
+        "multiple_testing_method": hierarchy_mod.MULTIPLE_TESTING_METHOD,
+        "notes": [
+            "sampling plans are frozen (hashed) before candidate outcomes are seen",
+            "inference units are container clusters, not frames/objects",
+            "baseline predictions are cached by dataset+model fingerprints",
+        ],
+    }

sensorflow/seqeval/attribution.py

@@ -0,0 +1,71 @@
+"""Regression attribution: turn node states into a decision-first regression map.
+
+On detection the question changes from "is there a regression?" to "WHERE and
+HOW BIG?". Every node with data yields a row carrying the affected class /
+condition / difficulty band, baseline and candidate values, absolute and
+relative deltas, the anytime-valid CI, raw and effective sample sizes, the
+e-values and the decision. The regression map is the REGRESSION-decided subset
+sorted by magnitude — the payload a release manager (or the safety Regression
+Gate) acts on.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List
+
+from sensorflow.seqeval.hierarchy import LEVELS, HierarchyController
+from sensorflow.seqeval.sequential import DECISION_REGRESSION
+
+
+def node_row(node, alpha_allocated: float) -> Dict:
+    t = node.test
+    clustering = node.clustering()
+    base_rate = t.sum_baseline / t.n_objects if t.n_objects else None
+    cand_rate = t.sum_candidate / t.n_objects if t.n_objects else None
+    abs_delta = None if base_rate is None else cand_rate - base_rate
+    rel_delta = (None if not base_rate else abs_delta / base_rate)
+    lo, hi = t.delta_interval()
+    return {
+        "node": node.key,
+        "level": node.level,
+        "level_name": LEVELS[node.level],
+        "safety_primary": node.safety_primary,
+        "suspect": node.suspect,
+        "metric": "recall",
+        "n": t.n_objects,
+        "n_clusters": clustering["n_clusters"],
+        "n_effective": clustering["n_effective"],
+        "icc": clustering["icc"],
+        "design_effect": clustering["design_effect"],
+        "baseline_value": None if base_rate is None else round(base_rate, 5),
+        "candidate_value": None if cand_rate is None else round(cand_rate, 5),
+        "abs_delta": None if abs_delta is None else round(abs_delta, 5),
+        "rel_delta": None if rel_delta is None else round(rel_delta, 5),
+        "delta_ci": [round(lo, 5), round(hi, 5)],
+        "e_regression": round(t.e_reg.e_value, 4),
+        "e_pass": round(t.e_pass.e_value, 4),
+        "bayes_p_regression": t.bayes_p_regression(),
+        "alpha_allocated": round(alpha_allocated, 6),
+        "decision": t.decision,
+        "decided_at_n": t.decided_at_n,
+        "discordant_pairs": {"regressions_n01": t.n01, "improvements_n10": t.n10},
+    }
+
+
+def build_attribution(hier: HierarchyController) -> Dict:
+    rows: List[Dict] = []
+    for node in hier.nodes.values():
+        if node.test.n_objects == 0:
+            continue
+        rows.append(node_row(node, hier.alpha_for(node)))
+    regressions = [r for r in rows if r["decision"] == DECISION_REGRESSION]
+    regressions.sort(key=lambda r: r["abs_delta"] if r["abs_delta"] is not None else 0.0)
+    affected_strata = [r["node"] for r in regressions if r["level"] == 3]
+    return {
+        "regression_map": regressions,
+        "affected_strata": affected_strata,
+        "all_nodes": sorted(rows, key=lambda r: (r["level"], r["node"])),
+        "note": ("regression_map lists nodes with a confirmed anytime-valid "
+                 "regression, most severe first; deltas are candidate minus "
+                 "baseline in absolute metric points"),
+    }

sensorflow/seqeval/controller.py

@@ -0,0 +1,565 @@
+"""Regression-controller state machine: sanity -> screening -> sequential
+confirmation -> targeted escalation, with CI/CD-style gate outcomes.
+
+Outcomes (three-outcome by construction, never "no news is good news"):
+    REGRESSION            -> gate "block": evidence + affected strata attached.
+    PASS                  -> gate "allow": overall AND every pre-registered
+                             safety primary reached the equivalence-style claim
+                             (drop confidently smaller than the margin).
+    INSUFFICIENT_EVIDENCE -> gate "expand_or_report": budget exhausted before
+                             either claim; reported with explicit
+                             "not proven equivalent" language.
+
+`evaluate_regression(...)` at the bottom is the clean entry point the safety
+Regression Gate (sensorflow/safety/) can call; the REST surface lives in
+api.py.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import threading
+import uuid
+from typing import Dict, List, Optional
+
+import numpy as np
+
+from sensorflow.megaeval import population as pop_mod
+from sensorflow.seqeval import attribution as attribution_mod
+from sensorflow.seqeval import hierarchy as hierarchy_mod
+from sensorflow.seqeval import ledger as ledger_mod
+from sensorflow.seqeval import paired as paired_mod
+from sensorflow.seqeval import planner as planner_mod
+from sensorflow.seqeval.sequential import (DECISION_INSUFFICIENT, DECISION_PASS,
+                                           DECISION_REGRESSION, approx_mde)
+
+DEFAULT_POLICY: Dict = {
+    "metric": "recall",
+    "delta_margin": 0.005,          # practical-significance margin, absolute points
+    "alpha": 0.05,                  # family alpha (regression direction)
+    "alpha_pass": 0.05,             # per-required-node equivalence alpha
+    "alpha_shares": dict(hierarchy_mod.DEFAULT_ALPHA_SHARES),
+    "condition_dim": "lighting",
+    "safety_primaries": ["pedestrian|night", "cyclist|night"],
+    "target_n": 24_000,
+    "min_per_stratum": 200,
+    "safety_floor": 2500,
+    "stage_fractions": [0.15, 0.40, 1.0],   # screening, confirmation, full plan
+    "batches_per_stage": 3,
+    "screening_e_threshold": 1.5,
+    "stop_on_regression": True,
+    "escalation": {"enabled": True, "max_extra_per_stratum": 4000,
+                   "batch_objects": 500},
+    "plan_seed": 12345,
+}
+
+GATE_FOR_DECISION = {
+    DECISION_REGRESSION: "block",
+    DECISION_PASS: "allow",
+    DECISION_INSUFFICIENT: "expand_or_report",
+}
+
+INSUFFICIENT_LANGUAGE = ("Budget exhausted before either claim could be made: the "
+                         "candidate is NOT PROVEN EQUIVALENT to the baseline within "
+                         "the practical margin, and no regression beyond the margin "
+                         "was confirmed. Do not treat this as a pass.")
+
+
+def merged_policy(overrides: Optional[Dict] = None) -> Dict:
+    policy = json.loads(json.dumps(DEFAULT_POLICY))
+    for k, v in (overrides or {}).items():
+        if isinstance(v, dict) and isinstance(policy.get(k), dict):
+            policy[k].update(v)
+        else:
+            policy[k] = v
+    return policy
+
+
+class SequentialRegressionRun:
+    def __init__(self, population_id: str, baseline: paired_mod.SimulatedModel,
+                 candidate: paired_mod.SimulatedModel, policy: Optional[Dict] = None,
+                 seed: Optional[int] = None):
+        self.run_id = f"seq-{uuid.uuid4().hex[:10]}"
+        self.population_id = population_id
+        self.baseline = baseline
+        self.candidate = candidate
+        self.policy = merged_policy(policy)
+        if seed is None:
+            seed = int(hashlib.sha256(
+                f"{population_id}|{baseline.fingerprint()}|{candidate.fingerprint()}"
+                .encode()).hexdigest()[:8], 16)
+        self.seed = int(seed)
+        self.status = "created"           # created|running|done|failed
+        self.stage = "pending"            # sanity|screening|sequential|escalation|done
+        self.decision = DECISION_INSUFFICIENT
+        self.stopping_reason: Optional[str] = None
+        self.error: Optional[str] = None
+        self.created_at = ledger_mod.now_iso()
+        self.finished_at: Optional[str] = None
+        self.samples_used = 0
+        self.escalation_used = 0
+        self.sanity: Dict = {}
+        self.plan: Optional[Dict] = None
+        self.hier: Optional[hierarchy_mod.HierarchyController] = None
+        self.attribution: Optional[Dict] = None
+        self.ledger = ledger_mod.EvidenceLedger(self.run_id)
+        self.message: Optional[str] = None
+
+    # ---------------------------------------------------------------- state
+
+    def gate(self) -> str:
+        return GATE_FOR_DECISION[self.decision]
+
+    def to_dict(self, include_trajectories: bool = True) -> Dict:
+        nodes = []
+        trajectories = {}
+        if self.hier is not None:
+            for key, node in self.hier.nodes.items():
+                if node.test.n_objects == 0:
+                    continue
+                snap = node.test.snapshot()
+                snap.update({"node": key, "level": node.level,
+                             "safety_primary": node.safety_primary,
+                             "suspect": node.suspect,
+                             "alpha_allocated": round(self.hier.alpha_for(node), 6),
+                             "e_threshold_regression": round(1.0 / self.hier.alpha_for(node), 2),
+                             "e_threshold_pass": round(1.0 / self.hier.alpha_pass, 2)})
+                nodes.append(snap)
+                if include_trajectories:
+                    trajectories[key] = {
+                        "points": node.test.trajectory,
+                        "boundaries": {
+                            "log_e_regression": round(float(np.log(1.0 / self.hier.alpha_for(node))), 4),
+                            "log_e_pass": round(float(np.log(1.0 / self.hier.alpha_pass)), 4),
+                            "delta_margin": -self.policy["delta_margin"],
+                        },
+                    }
+        plan_summary = None
+        if self.plan is not None:
+            plan_summary = {k: v for k, v in self.plan.items() if k != "_arrays"}
+        out = {
+            "run_id": self.run_id,
+            "population_id": self.population_id,
+            "baseline": self.baseline.to_dict(),
+            "candidate": self.candidate.to_dict(),
+            "status": self.status,
+            "stage": self.stage,
+            "decision": self.decision,
+            "gate": self.gate(),
+            "message": self.message,
+            "stopping_reason": self.stopping_reason,
+            "error": self.error,
+            "seed": self.seed,
+            "policy": self.policy,
+            "sanity": self.sanity,
+            "budget": {
+                "planned_total": plan_summary["total_allocated"] if plan_summary else None,
+                "samples_used": self.samples_used,
+                "escalation_used": self.escalation_used,
+                "full_population": plan_summary["population_objects"] if plan_summary else None,
+                "fraction_of_population": (round(self.samples_used / plan_summary["population_objects"], 5)
+                                           if plan_summary else None),
+            },
+            "plan": plan_summary,
+            "nodes": sorted(nodes, key=lambda r: (r["level"], r["node"])),
+            "created_at": self.created_at,
+            "finished_at": self.finished_at,
+        }
+        if include_trajectories:
+            out["trajectories"] = trajectories
+        return out
+
+
+class SeqevalStore:
+    def __init__(self):
+        self.lock = threading.RLock()
+        self.runs: Dict[str, SequentialRegressionRun] = {}
+        self._persisted: Dict[str, Dict] = {}
+        self._load_persisted()
+
+    def _load_persisted(self) -> None:
+        base = os.path.join(ledger_mod.seq_root(), "runs")
+        if not os.path.isdir(base):
+            return
+        for rid in sorted(os.listdir(base)):
+            path = os.path.join(base, rid, "run.json")
+            if not os.path.exists(path):
+                continue
+            try:
+                with open(path) as f:
+                    self._persisted[rid] = json.load(f)
+            except Exception:
+                continue
+
+    def create_run(self, population_id: str, baseline: Dict, candidate: Dict,
+                   policy: Optional[Dict] = None, seed: Optional[int] = None
+                   ) -> SequentialRegressionRun:
+        if pop_mod.load_meta(population_id) is None:
+            raise KeyError(f"Unknown population {population_id}")
+        run = SequentialRegressionRun(
+            population_id,
+            paired_mod.SimulatedModel(baseline.get("model_version", "baseline"),
+                                      baseline.get("effects")),
+            paired_mod.SimulatedModel(candidate.get("model_version", "candidate"),
+                                      candidate.get("effects")),
+            policy=policy, seed=seed)
+        with self.lock:
+            self.runs[run.run_id] = run
+        return run
+
+    def get_state(self, run_id: str, include_trajectories: bool = True) -> Optional[Dict]:
+        with self.lock:
+            run = self.runs.get(run_id)
+        if run is not None:
+            return run.to_dict(include_trajectories=include_trajectories)
+        state = self._persisted.get(run_id)
+        if state is not None and not include_trajectories:
+            state = {k: v for k, v in state.items() if k != "trajectories"}
+        return state
+
+    def list_states(self) -> List[Dict]:
+        seen = set()
+        out = []
+        with self.lock:
+            for run in self.runs.values():
+                seen.add(run.run_id)
+                out.append(run.to_dict(include_trajectories=False))
+        for rid, state in self._persisted.items():
+            if rid not in seen:
+                out.append({k: v for k, v in state.items() if k != "trajectories"})
+        out.sort(key=lambda s: s.get("created_at") or "", reverse=True)
+        return out
+
+    def save_run(self, run: SequentialRegressionRun) -> None:
+        d = ledger_mod.run_dir(run.run_id)
+        os.makedirs(d, exist_ok=True)
+        with open(os.path.join(d, "run.json"), "w") as f:
+            json.dump(run.to_dict(include_trajectories=True), f)
+
+    def start_async(self, run: SequentialRegressionRun) -> None:
+        threading.Thread(target=self.execute_sync, args=(run,), daemon=True).start()
+
+    # ---------------------------------------------------------------- engine
+
+    def execute_sync(self, run: SequentialRegressionRun) -> None:
+        try:
+            run.status = "running"
+            meta = pop_mod.load_meta(run.population_id)
+            policy = run.policy
+
+            # ---- stage: sanity + frozen plan ----------------------------
+            run.stage = "sanity"
+            plan = planner_mod.build_plan(
+                meta, target_n=policy["target_n"], seed=policy["plan_seed"],
+                condition_dim=policy["condition_dim"],
+                safety_primaries=policy["safety_primaries"],
+                min_per_stratum=policy["min_per_stratum"],
+                safety_floor=policy["safety_floor"])
+            run.plan = plan
+            frame = paired_mod.load_frame(meta)
+
+            all_ids = np.concatenate([plan["_arrays"][f"s{s}_ids"]
+                                      for s in sorted(int(k) for k in plan["strata"])])
+            smoke = all_ids[:min(500, all_ids.size)]
+            outcomes = paired_mod.paired_outcomes(meta, run.baseline, run.candidate,
+                                                  smoke, policy["condition_dim"])
+            b_rate = float(outcomes["baseline"].mean())
+            c_rate = float(outcomes["candidate"].mean())
+            discordance = float(np.mean(outcomes["baseline"] != outcomes["candidate"]))
+            run.sanity = {
+                "plan_hash": plan["plan_hash"],
+                "dataset_fingerprint": plan["dataset_fingerprint"],
+                "baseline_fingerprint": run.baseline.fingerprint(),
+                "candidate_fingerprint": run.candidate.fingerprint(),
+                "smoke_n": int(smoke.size),
+                "smoke_baseline_rate": round(b_rate, 4),
+                "smoke_candidate_rate": round(c_rate, 4),
+                "smoke_discordance": round(discordance, 4),
+                "ok": bool(0.2 < b_rate <= 1.0 and 0.2 < c_rate <= 1.0
+                           and discordance < 0.5),
+            }
+            if not run.sanity["ok"]:
+                run.decision = DECISION_INSUFFICIENT
+                run.stopping_reason = "sanity_failed"
+                run.message = ("Sanity stage failed (degenerate outcome rates or "
+                               "excessive discordance); evaluation aborted. "
+                               + INSUFFICIENT_LANGUAGE)
+                self._finalize(run)
+                return
+
+            # ---- build hierarchy + per-stratum cluster iterators ---------
+            n_cond = len(pop_mod.DIMENSIONS[policy["condition_dim"]])
+            hier = hierarchy_mod.HierarchyController(
+                plan["strata"], n_cond=n_cond, delta=policy["delta_margin"],
+                alpha=policy["alpha"], alpha_pass=policy["alpha_pass"],
+                alpha_shares=policy["alpha_shares"],
+                band_names=planner_mod.BAND_NAMES)
+            run.hier = hier
+
+            clusters = {int(s): planner_mod.stratum_clusters(plan, int(s))
+                        for s in plan["strata"]}
+            reserves = {int(s): planner_mod.stratum_clusters(plan, int(s), reserve=True)
+                        for s in plan["strata"]}
+            pointers = {s: 0 for s in clusters}
+            taken_objects = {s: 0 for s in clusters}
+            weights = {int(s): float(v["weight"]) for s, v in plan["strata"].items()}
+            key_to_sid = {f"stratum:{v['key']}": int(s) for s, v in plan["strata"].items()}
+            bands = np.digitize(frame["difficulty"],
+                                np.asarray(planner_mod.DIFFICULTY_BANDS)).astype(np.int8)
+
+            def make_batch(chunks: List) -> Optional[Dict[str, np.ndarray]]:
+                """chunks: list of (stratum_id, [cluster object_id arrays])."""
+                sids, ids = [], []
+                for s, cl in chunks:
+                    for arr in cl:
+                        ids.append(arr)
+                        sids.append(np.full(arr.size, s, dtype=np.int64))
+                if not ids:
+                    return None
+                oid = np.concatenate(ids)
+                sid = np.concatenate(sids)
+                oc = paired_mod.paired_outcomes(meta, run.baseline, run.candidate,
+                                                oid, policy["condition_dim"])
+                return {
+                    "object_id": oid,
+                    "container_id": frame["container_id"][oid],
+                    "class": frame["class"][oid].astype(np.int64),
+                    "cond": frame[policy["condition_dim"]][oid].astype(np.int64),
+                    "band": bands[oid].astype(np.int64),
+                    "weight": np.asarray([weights[s] for s in sid], dtype=np.float64),
+                    "d": oc["d"], "b": oc["baseline"], "c": oc["candidate"],
+                }
+
+            def process(chunks: List) -> None:
+                batch = make_batch(chunks)
+                if batch is None:
+                    return
+                hier.update_with_batch(batch)
+                hier.evaluate(policy["screening_e_threshold"])
+                hier.record_trajectories()
+                run.samples_used += int(batch["d"].size)
+
+            def take_until(s: int, target: int) -> List[np.ndarray]:
+                out = []
+                while pointers[s] < len(clusters[s]) and taken_objects[s] < target:
+                    arr = clusters[s][pointers[s]]
+                    out.append(arr)
+                    taken_objects[s] += arr.size
+                    pointers[s] += 1
+                return out
+
+            stopped = False
+
+            # ---- stages: screening then sequential confirmation ----------
+            for stage_idx, frac in enumerate(policy["stage_fractions"]):
+                run.stage = "screening" if stage_idx == 0 else "sequential"
+                for sub in range(policy["batches_per_stage"]):
+                    prev_frac = (policy["stage_fractions"][stage_idx - 1]
+                                 if stage_idx > 0 else 0.0)
+                    eff_frac = prev_frac + (frac - prev_frac) * (sub + 1) / policy["batches_per_stage"]
+                    chunks = []
+                    for s in clusters:
+                        target = int(np.ceil(eff_frac * plan["strata"][str(s)]["allocated"]))
+                        cl = take_until(s, target)
+                        if cl:
+                            chunks.append((s, cl))
+                    process(chunks)
+                    if policy["stop_on_regression"] and hier.any_regression():
+                        run.stopping_reason = "regression_confirmed"
+                        stopped = True
+                        break
+                    if hier.required_pass() and not hier.undecided_keys():
+                        run.stopping_reason = "all_passed"
+                        stopped = True
+                        break
+                if stopped:
+                    break
+
+            # ---- targeted escalation -------------------------------------
+            esc = policy["escalation"]
+            if not stopped and esc.get("enabled", True):
+                run.stage = "escalation"
+                for key in list(hier.undecided_keys()):
+                    node = hier.nodes.get(key)
+                    if node is None or node.level != 3:
+                        continue
+                    if not (node.suspect or node.safety_primary):
+                        continue
+                    s = key_to_sid.get(key)
+                    if s is None:
+                        continue
+                    pool = reserves.get(s, [])
+                    ptr, extra = 0, 0
+                    while (ptr < len(pool) and extra < esc["max_extra_per_stratum"]
+                           and node.test.decision == DECISION_INSUFFICIENT):
+                        chunk, size = [], 0
+                        while ptr < len(pool) and size < esc["batch_objects"]:
+                            chunk.append(pool[ptr])
+                            size += pool[ptr].size
+                            ptr += 1
+                        if not chunk:
+                            break
+                        process([(s, chunk)])
+                        extra += size
+                        run.escalation_used += size
+                        if policy["stop_on_regression"] and hier.any_regression():
+                            run.stopping_reason = "regression_confirmed"
+                            stopped = True
+                            break
+                    if stopped:
+                        break
+
+            # ---- finalize -------------------------------------------------
+            if run.stopping_reason is None:
+                run.stopping_reason = ("escalation_exhausted"
+                                       if esc.get("enabled", True) else "budget_exhausted")
+            if hier.any_regression():
+                run.decision = DECISION_REGRESSION
+                run.message = "Anytime-valid regression confirmed; see attribution."
+            elif hier.required_pass():
+                run.decision = DECISION_PASS
+                run.message = ("Overall and all pre-registered safety primaries proved "
+                               "equivalent within the practical margin.")
+            else:
+                run.decision = DECISION_INSUFFICIENT
+                run.message = INSUFFICIENT_LANGUAGE
+            run.attribution = attribution_mod.build_attribution(hier)
+            self._write_ledger(run)
+            self._finalize(run)
+        except Exception as e:  # pragma: no cover - defensive
+            run.status = "failed"
+            run.error = str(e)
+            run.finished_at = ledger_mod.now_iso()
+            try:
+                self.save_run(run)
+            except Exception:
+                pass
+
+    # ---------------------------------------------------------------- output
+
+    def _write_ledger(self, run: SequentialRegressionRun) -> None:
+        hier = run.hier
+        for node in hier.nodes.values():
+            t = node.test
+            if t.n_objects == 0:
+                continue
+            clustering = node.clustering()
+            nu = (t.n01 + t.n10) / t.n_objects if t.n_objects else 0.0
+            est = t.delta_estimate() or 0.0
+            var_d = max(nu - est ** 2, 1e-6)
+            base_rate = t.sum_baseline / t.n_objects
+            cand_rate = t.sum_candidate / t.n_objects
+            lo, hi = t.delta_interval()
+            alpha_alloc = hier.alpha_for(node)
+            run.ledger.append({
+                "run_id": run.run_id,
+                "baseline_model": run.baseline.to_dict(),
+                "candidate_model": run.candidate.to_dict(),
+                "dataset_version": run.population_id,
+                "seed": run.seed,
+                "stratum": node.key,
+                "level": node.level,
+                "metric": run.policy["metric"],
+                "baseline_value": round(base_rate, 6),
+                "candidate_value": round(cand_rate, 6),
+                "abs_delta": round(cand_rate - base_rate, 6),
+                "rel_delta": (round((cand_rate - base_rate) / base_rate, 6)
+                              if base_rate else None),
+                "delta_ci_low": round(lo, 6),
+                "delta_ci_high": round(hi, 6),
+                "mde_abs": approx_mde(var_d, clustering["n_effective"],
+                                      run.policy["delta_margin"], alpha_alloc),
+                "n": t.n_objects,
+                "n_effective": clustering["n_effective"],
+                "n_clusters": clustering["n_clusters"],
+                "design_effect": clustering["design_effect"],
+                "confidence_level": 1.0 - run.policy["alpha"],
+                "e_value": round(t.e_reg.e_value, 6),
+                "p_analogue": round(t.e_reg.p_analogue(), 8),
+                "bayes_p_regression": t.bayes_p_regression(),
+                "decision": t.decision,
+                "test_method": ledger_mod.TEST_METHOD,
+                "multiple_testing_method": hierarchy_mod.MULTIPLE_TESTING_METHOD,
+                "practical_margin": run.policy["delta_margin"],
+                "alpha_allocated": round(alpha_alloc, 6),
+                "stopping_reason": run.stopping_reason,
+                "timestamp": ledger_mod.now_iso(),
+            })
+        run.ledger.write_lineage({
+            "run_id": run.run_id,
+            "dataset_version": run.population_id,
+            "dataset_fingerprint": run.sanity.get("dataset_fingerprint"),
+            "plan_id": run.plan["plan_id"],
+            "plan_hash": run.plan["plan_hash"],
+            "seed": run.seed,
+            "statistical_config": run.policy,
+            "baseline_model": run.baseline.to_dict(),
+            "candidate_model": run.candidate.to_dict(),
+        })
+
+    def _finalize(self, run: SequentialRegressionRun) -> None:
+        run.status = "done"
+        run.stage = "done"
+        run.finished_at = ledger_mod.now_iso()
+        self.save_run(run)
+        # best-effort integration with platform accounting + audit trail
+        try:
+            from sensorflow.evaluation.process_units import ProcessMeter
+            from sensorflow.evaluation.records import get_store as get_legacy_store
+            legacy = get_legacy_store()
+            ProcessMeter(legacy, run.run_id).record("regression_tracking",
+                                                    run.samples_used)
+            legacy.audit("seqeval_decision", "seqeval_run", run.run_id,
+                         detail=f"{run.decision} ({run.stopping_reason}); "
+                                f"samples={run.samples_used}")
+            legacy.save()
+        except Exception:
+            pass
+
+
+_STORE: Optional[SeqevalStore] = None
+_STORE_LOCK = threading.Lock()
+
+
+def get_seqeval_store() -> SeqevalStore:
+    global _STORE
+    with _STORE_LOCK:
+        if _STORE is None:
+            _STORE = SeqevalStore()
+        return _STORE
+
+
+def reset_seqeval_store() -> None:
+    global _STORE
+    with _STORE_LOCK:
+        _STORE = None
+
+
+def evaluate_regression(population_id: str, baseline: Dict, candidate: Dict,
+                        policy: Optional[Dict] = None, seed: Optional[int] = None
+                        ) -> Dict:
+    """Clean entry point for the safety Regression Gate.
+
+    baseline/candidate: {"model_version": str, "effects": {stratum: pp_delta}}.
+    Returns a gate verdict with the evidence needed to block or allow:
+    {gate, decision, stopping_reason, samples_used, affected_strata,
+     regression_map, run_id, message}.
+    """
+    store = get_seqeval_store()
+    run = store.create_run(population_id, baseline, candidate, policy=policy, seed=seed)
+    store.execute_sync(run)
+    att = run.attribution or {"regression_map": [], "affected_strata": []}
+    return {
+        "run_id": run.run_id,
+        "gate": run.gate(),
+        "decision": run.decision,
+        "message": run.message,
+        "stopping_reason": run.stopping_reason,
+        "samples_used": run.samples_used,
+        "planned_total": run.plan["total_allocated"] if run.plan else None,
+        "full_population": run.plan["population_objects"] if run.plan else None,
+        "affected_strata": att["affected_strata"],
+        "regression_map": att["regression_map"],
+    }

sensorflow/seqeval/hierarchy.py

@@ -0,0 +1,223 @@
+"""Hierarchical gatekeeping multiple-testing controller.
+
+Four levels of hypotheses, each with its own alpha budget (budgets sum to the
+family alpha, so the union bound gives family-wise control across levels):
+
+    L1  overall population delta                (single test)
+    L2  per-class deltas                        (e-BH within level)
+    L3  per class x condition strata            (e-BH within level)
+    L4  difficulty bands (easy/medium/hard)     (e-BH within level)
+
+Within a level, discoveries are controlled with e-BH (Benjamini-Hochberg on
+e-values, Wang & Ramdas 2022): sort e-values descending and reject the k
+hypotheses with e_(k) >= M / (k * alpha_level). e-BH is valid under ARBITRARY
+dependence — essential here because the levels aggregate the same units and
+the strata share scenes — and composes cleanly with optional stopping because
+an e-process value at any stopping time is a valid e-value.
+
+Safety-critical strata are PRE-REGISTERED PRIMARIES: each gets its own reserved
+alpha and is tested individually, outside e-BH. They can therefore never be
+masked by an improving overall metric or by many well-behaved sibling strata
+(this is what makes the "overall improved, pedestrian-night regressed" case
+detectable by construction).
+
+Deltas are candidate-minus-baseline, so all rejection decisions are REGRESSION
+claims; PASS is the separate equivalence-style claim per node (e_pass at the
+node's pass alpha). Decisions are sticky: an anytime-valid rejection stays
+rejected as more data arrive.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List, Optional
+
+import numpy as np
+
+from sensorflow.seqeval import units as units_mod
+from sensorflow.seqeval.sequential import (DECISION_INSUFFICIENT, DECISION_PASS,
+                                           DECISION_REGRESSION, PairedSequentialTest)
+
+LEVELS = {1: "overall", 2: "class", 3: "stratum", 4: "difficulty"}
+
+# Default alpha budget shares (of the family alpha).
+DEFAULT_ALPHA_SHARES = {
+    "overall": 0.30,
+    "safety": 0.30,     # split equally among pre-registered safety primaries
+    "class": 0.15,
+    "stratum": 0.15,
+    "difficulty": 0.10,
+}
+
+MULTIPLE_TESTING_METHOD = ("hierarchical alpha budgeting (L1 overall / L2 class / "
+                           "L3 stratum / L4 difficulty) with e-BH within levels; "
+                           "safety primaries pre-registered at reserved alpha")
+
+
+class Node:
+    def __init__(self, key: str, level: int, delta: float, alpha_family: float,
+                 cls_code: Optional[int] = None, cond_code: Optional[int] = None,
+                 band: Optional[int] = None, weighted: bool = False,
+                 safety_primary: bool = False):
+        self.key = key
+        self.level = level
+        self.cls_code = cls_code
+        self.cond_code = cond_code
+        self.band = band
+        self.weighted = weighted
+        self.safety_primary = safety_primary
+        self.test = PairedSequentialTest(delta=delta, alpha=alpha_family)
+        self.suspect = False
+        # clustering diagnostics (accumulated per batch)
+        self._d_values: List[np.ndarray] = []
+        self._d_clusters: List[np.ndarray] = []
+
+    def select(self, batch: Dict[str, np.ndarray]) -> np.ndarray:
+        mask = np.ones(batch["d"].size, dtype=bool)
+        if self.cls_code is not None:
+            mask &= batch["class"] == self.cls_code
+        if self.cond_code is not None:
+            mask &= batch["cond"] == self.cond_code
+        if self.band is not None:
+            mask &= batch["band"] == self.band
+        return mask
+
+    def update(self, batch: Dict[str, np.ndarray]) -> None:
+        mask = self.select(batch)
+        if not mask.any():
+            return
+        d = batch["d"][mask]
+        containers = batch["container_id"][mask]
+        w = batch["weight"][mask] if self.weighted else None
+        means, _ = units_mod.cluster_units(d, containers, weights=w)
+        self.test.update_clusters(means)
+        self.test.record_objects(batch["b"][mask], batch["c"][mask])
+        self._d_values.append(d)
+        self._d_clusters.append(containers)
+
+    def clustering(self) -> Dict:
+        if not self._d_values:
+            return {"n": 0, "n_clusters": 0, "icc": 0.0, "design_effect": 1.0,
+                    "n_effective": 0.0, "mean_cluster_size": 0.0}
+        return units_mod.cluster_summary(np.concatenate(self._d_values),
+                                         np.concatenate(self._d_clusters))
+
+
+def e_bh_reject(e_values: Dict[str, float], alpha: float) -> List[str]:
+    """e-BH: reject the k hypotheses with largest e-values where
+    e_(k) >= M / (k * alpha). Valid under arbitrary dependence."""
+    if not e_values or alpha <= 0:
+        return []
+    items = sorted(e_values.items(), key=lambda kv: -kv[1])
+    m = len(items)
+    k_star = 0
+    for k, (_, e) in enumerate(items, start=1):
+        if e >= m / (k * alpha):
+            k_star = k
+    return [key for key, _ in items[:k_star]]
+
+
+class HierarchyController:
+    def __init__(self, strata: Dict[str, Dict], n_cond: int, delta: float,
+                 alpha: float, alpha_pass: float,
+                 alpha_shares: Optional[Dict[str, float]] = None,
+                 band_names: Optional[List[str]] = None):
+        self.alpha = float(alpha)
+        self.alpha_pass = float(alpha_pass)
+        self.shares = {**DEFAULT_ALPHA_SHARES, **(alpha_shares or {})}
+        self.nodes: Dict[str, Node] = {}
+        band_names = band_names or ["easy", "medium", "hard"]
+
+        self.nodes["overall"] = Node("overall", 1, delta, alpha, weighted=True)
+        classes = sorted({(int(s) // n_cond, v["class"]) for s, v in strata.items()})
+        for code, name in classes:
+            self.nodes[f"class:{name}"] = Node(f"class:{name}", 2, delta, alpha,
+                                               cls_code=code, weighted=True)
+        self.safety_keys: List[str] = []
+        for s, v in strata.items():
+            s = int(s)
+            key = f"stratum:{v['key']}"
+            node = Node(key, 3, delta, alpha, cls_code=s // n_cond,
+                        cond_code=s % n_cond, safety_primary=v.get("safety_primary", False))
+            self.nodes[key] = node
+            if node.safety_primary:
+                self.safety_keys.append(key)
+        for b, name in enumerate(band_names):
+            self.nodes[f"difficulty:{name}"] = Node(f"difficulty:{name}", 4, delta,
+                                                    alpha, band=b, weighted=True)
+
+    # ---- alpha allocation ------------------------------------------------
+
+    def alpha_for(self, node: Node) -> float:
+        a = self.alpha
+        if node.safety_primary:
+            return a * self.shares["safety"] / max(len(self.safety_keys), 1)
+        if node.level == 1:
+            return a * self.shares["overall"]
+        if node.level == 2:
+            return a * self.shares["class"]      # level budget, shared via e-BH
+        if node.level == 3:
+            return a * self.shares["stratum"]
+        return a * self.shares["difficulty"]
+
+    # ---- updates -----------------------------------------------------------
+
+    def update_with_batch(self, batch: Dict[str, np.ndarray]) -> None:
+        for node in self.nodes.values():
+            node.update(batch)
+
+    def evaluate(self, screening_e_threshold: float = 1.5) -> Dict[str, str]:
+        """Apply per-level testing; return {node_key: decision}. Sticky."""
+        # pre-registered primaries + single-hypothesis levels: individual tests
+        for node in self.nodes.values():
+            if node.safety_primary or node.level == 1:
+                node.test.evaluate(alpha_reg=self.alpha_for(node),
+                                   alpha_pass=self.alpha_pass)
+        # e-BH within L2 / L3(non-safety) / L4
+        for level, share_key in ((2, "class"), (3, "stratum"), (4, "difficulty")):
+            group = {k: n for k, n in self.nodes.items()
+                     if n.level == level and not n.safety_primary}
+            undecided = {k: n.test.e_reg.e_value for k, n in group.items()
+                         if n.test.decision == DECISION_INSUFFICIENT}
+            already = {k: n.test.e_reg.e_value for k, n in group.items()
+                       if n.test.decision == DECISION_REGRESSION}
+            # include already-rejected e-values so e-BH stays monotone/sticky
+            rejected = e_bh_reject({**undecided, **already},
+                                   self.alpha * self.shares[share_key])
+            for k in rejected:
+                node = group[k]
+                if node.test.decision == DECISION_INSUFFICIENT:
+                    node.test.decision = DECISION_REGRESSION
+                    node.test.decided_at_n = node.test.n_objects
+            # PASS claims are per-node equivalence statements
+            for k, n in group.items():
+                if n.test.decision == DECISION_INSUFFICIENT \
+                        and n.test.e_pass.e_value >= 1.0 / self.alpha_pass:
+                    n.test.decision = DECISION_PASS
+                    n.test.decided_at_n = n.test.n_objects
+        # screening flags (soft evidence, drives escalation priority)
+        for node in self.nodes.values():
+            if node.test.e_reg.e_value >= screening_e_threshold:
+                node.suspect = True
+        return {k: n.test.decision for k, n in self.nodes.items()}
+
+    def record_trajectories(self) -> None:
+        for node in self.nodes.values():
+            if node.test.n_objects > 0:
+                node.test.record_trajectory_point()
+
+    # ---- summaries -----------------------------------------------------
+
+    def decisions(self) -> Dict[str, str]:
+        return {k: n.test.decision for k, n in self.nodes.items()}
+
+    def any_regression(self) -> bool:
+        return any(n.test.decision == DECISION_REGRESSION for n in self.nodes.values())
+
+    def required_pass(self) -> bool:
+        """Overall + every safety primary must individually reach PASS."""
+        required = [self.nodes["overall"]] + [self.nodes[k] for k in self.safety_keys]
+        return all(n.test.decision == DECISION_PASS for n in required)
+
+    def undecided_keys(self) -> List[str]:
+        return [k for k, n in self.nodes.items()
+                if n.test.n_objects > 0 and n.test.decision == DECISION_INSUFFICIENT]

sensorflow/seqeval/ledger.py

@@ -0,0 +1,112 @@
+"""Machine-readable evidence ledger + seqeval storage root.
+
+Every stratum/node decision is persisted as one append-only JSONL record with
+every field an auditor needs to reproduce the decision: models, dataset
+version, seed, stratum, metric, baseline/candidate values, absolute+relative
+deltas, MDE at the achieved n, raw and effective sample sizes, confidence
+level, e-value and its p-analogue, the decision, the test and multiple-testing
+methods, the stopping reason and a timestamp. A lineage record (evaluator
+version + full statistical config) accompanies each run.
+
+Layout:
+    runs/seqeval/
+        plans/{plan_id}.json|.npz     frozen sampling plans (planner.py)
+        cache/{dataset_fp}-{model_fp}.npz   prediction cache (paired.py)
+        runs/{run_id}/run.json        controller state
+        runs/{run_id}/ledger.jsonl    evidence records
+        runs/{run_id}/lineage.json    evaluator + statistical config lineage
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+SEQEVAL_VERSION = "seqeval-1.0.0"
+
+_SEQ_ROOT = os.path.join("runs", "seqeval")
+_ROOT_LOCK = threading.Lock()
+
+
+def seq_root() -> str:
+    return _SEQ_ROOT
+
+
+def set_seqeval_root(path: str) -> None:
+    """Test hook: relocate all seqeval storage."""
+    global _SEQ_ROOT
+    with _ROOT_LOCK:
+        _SEQ_ROOT = str(path)
+
+
+def run_dir(run_id: str) -> str:
+    return os.path.join(seq_root(), "runs", run_id)
+
+
+def now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+REQUIRED_FIELDS = [
+    "baseline_model", "candidate_model", "dataset_version", "seed",
+    "stratum", "level", "metric",
+    "baseline_value", "candidate_value", "abs_delta", "rel_delta",
+    "delta_ci_low", "delta_ci_high", "mde_abs",
+    "n", "n_effective", "n_clusters", "design_effect",
+    "confidence_level", "e_value", "p_analogue", "bayes_p_regression",
+    "decision", "test_method", "multiple_testing_method",
+    "practical_margin", "alpha_allocated",
+    "stopping_reason", "timestamp",
+]
+
+TEST_METHOD = ("paired cluster-mean empirical-Bernstein confidence sequence + "
+               "one-sided betting e-process (anytime-valid)")
+
+
+class EvidenceLedger:
+    """Append-only JSONL evidence store for one controller run."""
+
+    def __init__(self, run_id: str):
+        self.run_id = run_id
+        self.dir = run_dir(run_id)
+        os.makedirs(self.dir, exist_ok=True)
+        self.path = os.path.join(self.dir, "ledger.jsonl")
+        self._lock = threading.Lock()
+
+    def append(self, record: Dict) -> Dict:
+        missing = [f for f in REQUIRED_FIELDS if f not in record]
+        if missing:
+            raise ValueError(f"Evidence record missing fields: {missing}")
+        record = {**record, "record_type": record.get("record_type", "node_decision")}
+        with self._lock:
+            with open(self.path, "a") as f:
+                f.write(json.dumps(record) + "\n")
+        return record
+
+    def records(self) -> List[Dict]:
+        if not os.path.exists(self.path):
+            return []
+        out = []
+        with open(self.path) as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    out.append(json.loads(line))
+        return out
+
+    def write_lineage(self, lineage: Dict) -> Dict:
+        lineage = {"evaluator_version": SEQEVAL_VERSION,
+                   "timestamp": now_iso(), **lineage}
+        with open(os.path.join(self.dir, "lineage.json"), "w") as f:
+            json.dump(lineage, f, indent=1)
+        return lineage
+
+    def lineage(self) -> Optional[Dict]:
+        path = os.path.join(self.dir, "lineage.json")
+        if not os.path.exists(path):
+            return None
+        with open(path) as f:
+            return json.load(f)

sensorflow/seqeval/paired.py

@@ -0,0 +1,267 @@
+"""Paired evaluation harness with a fingerprint-keyed prediction cache.
+
+Core design: the SAME sampled units are run through both models. The paired
+difference d_i = c_i - b_i removes the between-unit difficulty variance that
+dominates two-independent-sample comparisons (Var(d) ~ discordance rate, an
+order of magnitude below p(1-p) for adjacent model versions), which is where
+most of the sample-size savings in this system come from.
+
+Models are simulated through the megaeval synthetic machinery's population
+(dimension codes + difficulty + container structure) with configurable
+per-stratum performance effects, so a "2pp pedestrian-night regression"
+candidate is constructible deterministically:
+
+    SimulatedModel("cand-v2", effects={"pedestrian|night": -0.02, "__global__": 0.004})
+
+Determinism/pairing:
+  * a per-OBJECT pairing latent u_i drawn once per dataset (seeded by the
+    population seed) is shared by every model -> outcomes are paired;
+  * each model adds a small independent "flip" noise stream seeded by its own
+    fingerprint -> realistic discordance even at Delta = 0;
+  * a container-level scene effect (shared by both models) induces honest
+    intra-cluster correlation for the clustering machinery to handle.
+
+Prediction cache: full-population success vectors keyed by
+(dataset fingerprint, model fingerprint) and persisted under
+runs/seqeval/cache/. Baseline predictions are computed once and reused across
+candidate updates; COMPUTE_COUNTS lets tests assert no recompute happened.
+Invalidation is by construction: a new dataset or new checkpoint changes the
+fingerprint and therefore the key; stale entries never collide.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import threading
+from typing import Dict, Optional, Tuple
+
+import numpy as np
+
+from sensorflow.megaeval import population as pop_mod
+from sensorflow.seqeval import ledger as ledger_mod
+
+# Simulation profile (baseline physics of the synthetic detector).
+BASE_DETECT = 0.945
+DIFFICULTY_WEIGHT = 0.16
+NIGHT_PENALTY = 0.030
+VRU_PENALTY = 0.020
+SCENE_EFFECT_SD = 0.030
+FLIP_NOISE = 0.04          # per-model chance of drawing a fresh latent
+VRU_CLASSES = (1, 2, 3)    # pedestrian, cyclist, motorcycle
+
+GLOBAL_EFFECT_KEY = "__global__"
+
+# fingerprint -> number of actual (non-cache) prediction computations
+COMPUTE_COUNTS: Dict[str, int] = {}
+_COUNT_LOCK = threading.Lock()
+
+
+class SimulatedModel:
+    """A model version plus per-stratum performance effects.
+
+    effects keys: "class|condition" (e.g. "pedestrian|night") in absolute
+    detection-probability points, or "__global__" applied everywhere.
+    """
+
+    def __init__(self, model_version: str, effects: Optional[Dict[str, float]] = None):
+        self.model_version = model_version
+        self.effects = dict(effects or {})
+
+    def fingerprint(self) -> str:
+        blob = json.dumps({"v": self.model_version,
+                           "e": {k: round(float(v), 6)
+                                 for k, v in sorted(self.effects.items())}},
+                          sort_keys=True)
+        return hashlib.sha256(blob.encode()).hexdigest()[:16]
+
+    def to_dict(self) -> Dict:
+        return {"model_version": self.model_version, "effects": self.effects,
+                "fingerprint": self.fingerprint()}
+
+
+def dataset_fingerprint(meta: Dict) -> str:
+    blob = json.dumps({"pop": meta["population_id"], "seed": meta["seed"],
+                       "n": meta["num_objects"]}, sort_keys=True)
+    return hashlib.sha256(blob.encode()).hexdigest()[:16]
+
+
+# ------------------------------------------------------------------ frame
+
+
+_FRAME_CACHE: Dict[str, Dict[str, np.ndarray]] = {}
+_FRAME_LOCK = threading.Lock()
+
+
+def load_frame(meta: Dict) -> Dict[str, np.ndarray]:
+    """Full-population sampling frame (dimension codes only), sorted by
+    object_id so that object_id == row index. Cached in memory per population."""
+    pid = meta["population_id"]
+    with _FRAME_LOCK:
+        if pid in _FRAME_CACHE:
+            return _FRAME_CACHE[pid]
+    cols_needed = ["object_id", "container_id", "class", "lighting", "scenario",
+                   "weather", "difficulty", "safety_critical"]
+    parts = {k: [] for k in cols_needed}
+    for p in range(meta["num_partitions"]):
+        part = pop_mod.load_partition(pid, p)
+        for k in cols_needed:
+            parts[k].append(part[k])
+    frame = {k: np.concatenate(v) for k, v in parts.items()}
+    order = np.argsort(frame["object_id"], kind="stable")
+    frame = {k: v[order] for k, v in frame.items()}
+    with _FRAME_LOCK:
+        if len(_FRAME_CACHE) > 4:
+            _FRAME_CACHE.clear()
+        _FRAME_CACHE[pid] = frame
+    return frame
+
+
+# ------------------------------------------------------------------ latents
+
+
+def _pair_latents(meta: Dict) -> np.ndarray:
+    """Per-object pairing latent, shared by all models on this dataset."""
+    rng = np.random.default_rng([int(meta["seed"]), 0x5E9, 424242])
+    return rng.random(int(meta["num_objects"]))
+
+
+def _scene_effects(meta: Dict) -> np.ndarray:
+    """Container-level 'scene hardness' shared by all models (drives ICC)."""
+    rng = np.random.default_rng([int(meta["seed"]), 777])
+    return rng.normal(0.0, SCENE_EFFECT_SD, size=int(meta["num_containers"]))
+
+
+_LATENT_CACHE: Dict[str, Tuple[np.ndarray, np.ndarray]] = {}
+
+
+def _latents(meta: Dict) -> Tuple[np.ndarray, np.ndarray]:
+    pid = meta["population_id"]
+    if pid not in _LATENT_CACHE:
+        if len(_LATENT_CACHE) > 4:
+            _LATENT_CACHE.clear()
+        _LATENT_CACHE[pid] = (_pair_latents(meta), _scene_effects(meta))
+    return _LATENT_CACHE[pid]
+
+
+# ------------------------------------------------------------------ prediction
+
+
+def _detect_probability(frame: Dict[str, np.ndarray], meta: Dict,
+                        model: SimulatedModel, condition_dim: str) -> np.ndarray:
+    difficulty = frame["difficulty"].astype(np.float64)
+    p = np.full(difficulty.shape, BASE_DETECT)
+    p -= DIFFICULTY_WEIGHT * difficulty
+    p -= NIGHT_PENALTY * (frame["lighting"] == 1)
+    p -= VRU_PENALTY * np.isin(frame["class"], VRU_CLASSES)
+    _, scene = _latents(meta)
+    p += scene[frame["container_id"]]
+    p += float(model.effects.get(GLOBAL_EFFECT_KEY, 0.0))
+    cond_vocab = pop_mod.DIMENSIONS[condition_dim]
+    cls_vocab = pop_mod.DIMENSIONS["class"]
+    for key, eff in model.effects.items():
+        if key == GLOBAL_EFFECT_KEY or "|" not in key:
+            continue
+        cls_name, cond_name = key.split("|", 1)
+        if cls_name not in cls_vocab or cond_name not in cond_vocab:
+            continue
+        mask = ((frame["class"] == cls_vocab.index(cls_name))
+                & (frame[condition_dim] == cond_vocab.index(cond_name)))
+        p[mask] += float(eff)
+    return np.clip(p, 0.01, 0.995)
+
+
+def _compute_predictions(meta: Dict, model: SimulatedModel,
+                         condition_dim: str) -> np.ndarray:
+    frame = load_frame(meta)
+    p = _detect_probability(frame, meta, model, condition_dim)
+    u_pair, _ = _latents(meta)
+    fp_int = int(model.fingerprint(), 16)
+    rng = np.random.default_rng([fp_int % (2**32), int(meta["seed"]), 7])
+    n = u_pair.size
+    swap = rng.random(n) < FLIP_NOISE
+    fresh = rng.random(n)
+    u = np.where(swap, fresh, u_pair)
+    return (u < p).astype(np.uint8)
+
+
+class PredictionCache:
+    """Disk + memory cache of full-population success vectors.
+
+    Key = (dataset fingerprint, model fingerprint). Both fingerprints cover
+    everything the predictions depend on, so invalidation is automatic: any
+    change to the dataset (new population/seed/size) or the model (version or
+    effects) produces a new key.
+    """
+
+    def __init__(self):
+        self._mem: Dict[str, np.ndarray] = {}
+        self._lock = threading.Lock()
+
+    def _path(self, key: str) -> str:
+        return os.path.join(ledger_mod.seq_root(), "cache", f"{key}.npz")
+
+    def get_or_compute(self, meta: Dict, model: SimulatedModel,
+                       condition_dim: str = "lighting") -> np.ndarray:
+        key = f"{dataset_fingerprint(meta)}-{model.fingerprint()}"
+        with self._lock:
+            if key in self._mem:
+                return self._mem[key]
+        path = self._path(key)
+        if os.path.exists(path):
+            with np.load(path) as z:
+                success = z["success"]
+        else:
+            success = _compute_predictions(meta, model, condition_dim)
+            with _COUNT_LOCK:
+                COMPUTE_COUNTS[key] = COMPUTE_COUNTS.get(key, 0) + 1
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+            np.savez_compressed(path, success=success,
+                                model_version=np.array([model.model_version]))
+        with self._lock:
+            if len(self._mem) > 16:
+                self._mem.clear()
+            self._mem[key] = success
+        return success
+
+    def clear_memory(self) -> None:
+        with self._lock:
+            self._mem.clear()
+
+
+_CACHE: Optional[PredictionCache] = None
+_CACHE_LOCK = threading.Lock()
+
+
+def get_prediction_cache() -> PredictionCache:
+    global _CACHE
+    with _CACHE_LOCK:
+        if _CACHE is None:
+            _CACHE = PredictionCache()
+        return _CACHE
+
+
+def reset_prediction_cache() -> None:
+    global _CACHE
+    with _CACHE_LOCK:
+        _CACHE = None
+    with _FRAME_LOCK:
+        _FRAME_CACHE.clear()
+    _LATENT_CACHE.clear()
+    with _COUNT_LOCK:
+        COMPUTE_COUNTS.clear()
+
+
+def paired_outcomes(meta: Dict, baseline: SimulatedModel, candidate: SimulatedModel,
+                    object_ids: np.ndarray, condition_dim: str = "lighting"
+                    ) -> Dict[str, np.ndarray]:
+    """Per-unit paired outcomes for the given objects (baseline served from
+    cache; the same sampled units go through both models)."""
+    cache = get_prediction_cache()
+    b_full = cache.get_or_compute(meta, baseline, condition_dim)
+    c_full = cache.get_or_compute(meta, candidate, condition_dim)
+    idx = np.asarray(object_ids, dtype=np.int64)
+    b = b_full[idx].astype(np.int8)
+    c = c_full[idx].astype(np.int8)
+    return {"baseline": b, "candidate": c, "d": (c - b).astype(np.float64)}

sensorflow/seqeval/planner.py

@@ -0,0 +1,227 @@
+"""Frozen stratified sampling planner (hybrid Neyman + risk floors + safety minimums).
+
+The plan is built from the POPULATION ONLY (dimension codes, counts, and a
+candidate-independent prior on baseline variance) and is persisted with its
+seed, config and a content hash BEFORE any candidate outcome is observed.
+This is the sampling-bias firewall: no outcome-dependent selection is possible
+because the selection provably cannot depend on outcomes (the planner never
+sees them), and the hash gives auditors a cheap way to verify the plan did not
+change after results started arriving.
+
+Allocation (hybrid, in order):
+  1. Neyman baseline: n_h proportional to N_h * sigma_h, where sigma_h =
+     sqrt(p_h (1 - p_h)) uses a PRIOR per-stratum baseline success rate
+     (deterministic profile; never candidate data).
+  2. Risk weighting: VRU-class and night/adverse-condition strata get a
+     multiplicative boost (they are where regressions hurt).
+  3. Floors: every stratum gets a mandatory minimum; pre-registered
+     safety-critical strata get a much larger minimum sized for the target MDE.
+  4. Truncation at stratum population size; whole CLUSTERS (container groups)
+     are selected so the inference unit stays intact.
+
+Because allocation is non-proportional, every sampled object carries the
+Horvitz-Thompson style weight w_h = N_h / n_h; population-level estimates are
+weight-corrected and therefore remain unbiased despite oversampling.
+
+The plan also stores a per-stratum escalation RESERVE (the remaining cluster
+units in shuffled order) so the controller can expand the sample without
+breaking the frozen-selection property: the reserve order was fixed at plan
+time too.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from typing import Dict, List, Optional
+
+import numpy as np
+
+from sensorflow.megaeval import population as pop_mod
+from sensorflow.seqeval import ledger as ledger_mod
+from sensorflow.seqeval import paired as paired_mod
+
+DIFFICULTY_BANDS = [0.30, 0.55]  # easy < 0.30 <= medium < 0.55 <= hard
+BAND_NAMES = ["easy", "medium", "hard"]
+
+# Candidate-independent prior baseline success rates used for Neyman sigma_h.
+_PRIOR_BASE = 0.90
+_PRIOR_NIGHT_DROP = 0.04
+_PRIOR_VRU_DROP = 0.03
+
+
+def stratum_key(cls_name: str, cond_name: str) -> str:
+    return f"{cls_name}|{cond_name}"
+
+
+def _prior_rate(cls_code: int, cond_code: int, condition_dim: str) -> float:
+    p = _PRIOR_BASE
+    if condition_dim == "lighting" and cond_code == 1:
+        p -= _PRIOR_NIGHT_DROP
+    if cls_code in paired_mod.VRU_CLASSES:
+        p -= _PRIOR_VRU_DROP
+    return p
+
+
+def _risk_weight(cls_code: int, cond_code: int, condition_dim: str) -> float:
+    w = 1.0
+    if cls_code in paired_mod.VRU_CLASSES:
+        w *= 1.5
+    if condition_dim == "lighting" and cond_code == 1:
+        w *= 1.3
+    return w
+
+
+def plan_path(plan_id: str) -> str:
+    return os.path.join(ledger_mod.seq_root(), "plans", plan_id)
+
+
+def build_plan(meta: Dict, target_n: int, seed: int,
+               condition_dim: str = "lighting",
+               safety_primaries: Optional[List[str]] = None,
+               min_per_stratum: int = 200,
+               safety_floor: int = 2500,
+               persist: bool = True) -> Dict:
+    """Build (and persist) a frozen stratified cluster-sampling plan.
+
+    Takes NO candidate/model arguments by design — see module docstring.
+    """
+    safety_primaries = list(safety_primaries or [])
+    frame = paired_mod.load_frame(meta)
+    cls_vocab = pop_mod.DIMENSIONS["class"]
+    cond_vocab = pop_mod.DIMENSIONS[condition_dim]
+    n_cond = len(cond_vocab)
+    strata_ids = frame["class"].astype(np.int64) * n_cond + frame[condition_dim].astype(np.int64)
+    n_strata = len(cls_vocab) * n_cond
+
+    counts = np.bincount(strata_ids, minlength=n_strata)
+    keys = [stratum_key(cls_vocab[s // n_cond], cond_vocab[s % n_cond])
+            for s in range(n_strata)]
+
+    # --- 1-2. Neyman shares with risk multipliers (prior variance, no outcomes)
+    shares = np.zeros(n_strata)
+    for s in range(n_strata):
+        if counts[s] == 0:
+            continue
+        p = _prior_rate(s // n_cond, s % n_cond, condition_dim)
+        sigma = np.sqrt(p * (1 - p))
+        shares[s] = counts[s] * sigma * _risk_weight(s // n_cond, s % n_cond, condition_dim)
+    shares = shares / shares.sum() if shares.sum() > 0 else shares
+
+    # --- 3. floors and safety minimums, truncated at N_h
+    alloc = np.zeros(n_strata, dtype=np.int64)
+    for s in range(n_strata):
+        if counts[s] == 0:
+            continue
+        n_h = int(round(target_n * shares[s]))
+        n_h = max(n_h, min(min_per_stratum, int(counts[s])))
+        if keys[s] in safety_primaries:
+            n_h = max(n_h, min(safety_floor, int(0.85 * counts[s])))
+        alloc[s] = min(n_h, int(counts[s]))
+
+    # --- 4. whole-cluster selection in seeded shuffled order + reserve
+    rng = np.random.default_rng([int(seed), int(meta["seed"]), 20260812])
+    strata = {}
+    arrays = {}
+    for s in range(n_strata):
+        if counts[s] == 0:
+            continue
+        rows = np.where(strata_ids == s)[0]
+        containers = frame["container_id"][rows]
+        order = np.argsort(containers, kind="stable")
+        rows, containers = rows[order], containers[order]
+        uniq, start = np.unique(containers, return_index=True)
+        bounds = np.append(start, rows.size)
+        cluster_order = rng.permutation(uniq.size)
+
+        sel_ids, sel_bounds = [], [0]
+        res_ids, res_bounds = [], [0]
+        taken = 0
+        for ci in cluster_order:
+            members = frame["object_id"][rows[bounds[ci]:bounds[ci + 1]]]
+            if taken < alloc[s]:
+                sel_ids.append(members)
+                sel_bounds.append(sel_bounds[-1] + members.size)
+                taken += members.size
+            else:
+                res_ids.append(members)
+                res_bounds.append(res_bounds[-1] + members.size)
+        sel = np.concatenate(sel_ids) if sel_ids else np.empty(0, dtype=np.int64)
+        res = np.concatenate(res_ids) if res_ids else np.empty(0, dtype=np.int64)
+        arrays[f"s{s}_ids"] = sel
+        arrays[f"s{s}_bounds"] = np.asarray(sel_bounds, dtype=np.int64)
+        arrays[f"s{s}_res_ids"] = res
+        arrays[f"s{s}_res_bounds"] = np.asarray(res_bounds, dtype=np.int64)
+        strata[str(s)] = {
+            "key": keys[s],
+            "class": cls_vocab[s // n_cond],
+            "condition": cond_vocab[s % n_cond],
+            "N": int(counts[s]),
+            "allocated": int(sel.size),
+            "clusters": len(sel_bounds) - 1,
+            "reserve": int(res.size),
+            "weight": round(float(counts[s]) / max(sel.size, 1), 4),
+            "safety_primary": keys[s] in safety_primaries,
+        }
+
+    config = {
+        "target_n": int(target_n),
+        "condition_dim": condition_dim,
+        "safety_primaries": safety_primaries,
+        "min_per_stratum": int(min_per_stratum),
+        "safety_floor": int(safety_floor),
+        "difficulty_bands": DIFFICULTY_BANDS,
+        "allocation": "neyman(prior sigma) * risk_weights, floors, whole clusters",
+    }
+    h = hashlib.sha256()
+    h.update(json.dumps({"dataset": paired_mod.dataset_fingerprint(meta),
+                         "seed": int(seed), "config": config}, sort_keys=True).encode())
+    for s in sorted(int(k) for k in strata):
+        h.update(arrays[f"s{s}_ids"].tobytes())
+        h.update(arrays[f"s{s}_res_ids"].tobytes())
+    plan_hash = h.hexdigest()[:24]
+
+    plan = {
+        "plan_id": f"plan-{plan_hash[:10]}",
+        "plan_hash": plan_hash,
+        "population_id": meta["population_id"],
+        "dataset_fingerprint": paired_mod.dataset_fingerprint(meta),
+        "seed": int(seed),
+        "created_at": ledger_mod.now_iso(),
+        "config": config,
+        "population_objects": int(meta["num_objects"]),
+        "total_allocated": int(sum(v["allocated"] for v in strata.values())),
+        "strata": strata,
+        "frozen_before_candidate_outcomes": True,
+    }
+    if persist:
+        d = plan_path(plan["plan_id"])
+        os.makedirs(d, exist_ok=True)
+        with open(os.path.join(d, "plan.json"), "w") as f:
+            json.dump(plan, f)
+        np.savez_compressed(os.path.join(d, "units.npz"), **arrays)
+    plan["_arrays"] = arrays  # in-memory only; not serialized
+    return plan
+
+
+def load_plan(plan_id: str) -> Optional[Dict]:
+    d = plan_path(plan_id)
+    meta_path = os.path.join(d, "plan.json")
+    if not os.path.exists(meta_path):
+        return None
+    with open(meta_path) as f:
+        plan = json.load(f)
+    with np.load(os.path.join(d, "units.npz")) as z:
+        plan["_arrays"] = {k: z[k] for k in z.files}
+    return plan
+
+
+def stratum_clusters(plan: Dict, s: int, reserve: bool = False) -> List[np.ndarray]:
+    """Ordered list of cluster-unit object_id arrays for stratum s."""
+    suffix = "_res" if reserve else ""
+    ids = plan["_arrays"].get(f"s{s}{suffix}_ids")
+    bounds = plan["_arrays"].get(f"s{s}{suffix}_bounds")
+    if ids is None or bounds is None or ids.size == 0:
+        return []
+    return [ids[bounds[i]:bounds[i + 1]] for i in range(bounds.size - 1)]

sensorflow/seqeval/sequential.py

@@ -0,0 +1,286 @@
+"""Anytime-valid sequential inference for paired mean differences.
+
+Why not repeated fixed-sample tests: a CLT/Wilson interval is valid at ONE
+pre-committed n. Recomputing it after every batch and stopping on the first
+significant look inflates type-I error without bound (the law of the iterated
+logarithm guarantees an eventual false crossing). Everything here is therefore
+ANYTIME-VALID: the guarantee holds simultaneously over all sample sizes and
+data-dependent stopping rules.
+
+Machinery (Waudby-Smith & Ramdas, "Estimating means of bounded random variables
+by betting", JRSS-B 2023):
+
+  * `EmpiricalBernsteinCS` — predictable plug-in empirical-Bernstein confidence
+    sequence for a mean of [0,1]-bounded observations. Variance-adaptive: for
+    paired differences (variance << 1/4) it is far tighter than Hoeffding.
+  * `OneSidedEProcess` — a betting supermartingale (e-process) against a
+    one-sided null "mean >= m" or "mean <= m". Ville's inequality gives
+    P(e-value ever >= 1/alpha) <= alpha under the null, at any stopping time.
+  * `PairedSequentialTest` — combines both on paired differences d in [-1,1]
+    (mapped to x=(d+1)/2), producing the mandatory THREE-OUTCOME decision
+    against a practical-significance margin delta:
+        REGRESSION            e-process against  H0: Delta >= -delta  crossed 1/alpha_reg
+        PASS                  e-process against  H0: Delta <= -delta  crossed 1/alpha_pass
+                              (an equivalence-style claim: the drop, if any, is
+                               confidently smaller than the margin)
+        INSUFFICIENT_EVIDENCE neither — never interpreted as proven equivalence.
+  * A Beta-Binomial posterior P(Delta < -delta | data) is computed alongside as
+    an operational, human-readable complement (it is NOT the gate criterion).
+
+Observations fed in are CLUSTER means (see units.py), so temporal/spatial
+correlation is handled by construction rather than by post-hoc correction.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Dict, List, Optional
+
+from scipy import stats as _sstats
+
+DECISION_REGRESSION = "REGRESSION"
+DECISION_PASS = "PASS"
+DECISION_INSUFFICIENT = "INSUFFICIENT_EVIDENCE"
+
+
+class EmpiricalBernsteinCS:
+    """Predictable plug-in empirical-Bernstein confidence sequence on [0,1].
+
+    Running-intersection bounds are kept (intersecting a CS over time preserves
+    coverage), so the interval never widens as data accrue.
+    """
+
+    def __init__(self, alpha: float = 0.05, truncation: float = 0.5):
+        self.alpha = alpha
+        self.truncation = truncation
+        self.t = 0
+        self._sum_x = 0.0
+        self._mu_hat = 0.5           # mu_hat_0 = 1/2 (prior-regularized running mean)
+        self._var_acc = 0.25         # 1/4 + sum_i (x_i - mu_hat_i)^2
+        self._sigma2_hat = 0.25
+        self._sum_lam = 0.0
+        self._sum_lam_x = 0.0
+        self._sum_v_psi = 0.0
+        self.lower = 0.0
+        self.upper = 1.0
+
+    def update(self, x: float) -> None:
+        x = min(1.0, max(0.0, float(x)))
+        t = self.t + 1
+        log2a = math.log(2.0 / self.alpha)
+        lam = math.sqrt(2.0 * log2a / (max(self._sigma2_hat, 1e-12) * t * math.log1p(t)))
+        lam = min(lam, self.truncation)
+        v = 4.0 * (x - self._mu_hat) ** 2
+        psi = (-math.log1p(-lam) - lam) / 4.0
+        self._sum_lam += lam
+        self._sum_lam_x += lam * x
+        self._sum_v_psi += v * psi
+        # advance predictable estimates AFTER betting with the old ones
+        self.t = t
+        self._sum_x += x
+        self._mu_hat = (0.5 + self._sum_x) / (t + 1)
+        self._var_acc += (x - self._mu_hat) ** 2
+        self._sigma2_hat = self._var_acc / (t + 1)
+        if self._sum_lam > 0:
+            center = self._sum_lam_x / self._sum_lam
+            radius = (log2a + self._sum_v_psi) / self._sum_lam
+            self.lower = max(self.lower, center - radius)
+            self.upper = min(self.upper, center + radius)
+
+    def interval(self) -> tuple:
+        return (max(0.0, self.lower), min(1.0, self.upper))
+
+
+class OneSidedEProcess:
+    """Betting e-process for a one-sided mean null on [0,1] observations.
+
+    side="below": H0: mu >= m, wealth grows when observations fall below m.
+    side="above": H0: mu <= m, wealth grows when observations exceed m.
+
+    Bet sizing: max of the empirical-Bernstein recipe (drives early growth
+    while the sample variance estimate is still coarse) and an aGRAPA /
+    approximate-Kelly bet lambda ~ drift / (sigma^2 + drift^2) (near-optimal
+    asymptotic growth rate ~ KL divergence per observation once a real drift
+    emerges). Both are predictable and clipped to the admissible range, so the
+    wealth process remains a nonnegative supermartingale under the null — the
+    sizing choice affects power only, never validity.
+    """
+
+    def __init__(self, m: float, side: str, alpha_ref: float = 0.05,
+                 truncation: float = 0.75):
+        assert side in ("below", "above")
+        self.m = float(m)
+        self.side = side
+        self.alpha_ref = alpha_ref
+        self.truncation = truncation
+        self.t = 0
+        self.log_e = 0.0
+        self._sum_x = 0.0
+        self._mu_hat = 0.5
+        self._var_acc = 0.25
+        self._sigma2_hat = 0.25
+
+    def update(self, x: float) -> None:
+        x = min(1.0, max(0.0, float(x)))
+        t = self.t + 1
+        loga = math.log(1.0 / self.alpha_ref)
+        sigma2 = max(self._sigma2_hat, 1e-12)
+        lam_eb = math.sqrt(2.0 * loga / (sigma2 * t * math.log1p(t)))
+        drift = (self.m - self._mu_hat) if self.side == "below" else (self._mu_hat - self.m)
+        lam_kelly = max(drift, 0.0) / (sigma2 + drift * drift)
+        lam = max(lam_eb, lam_kelly)
+        if self.side == "below":
+            lam = min(lam, self.truncation / max(1.0 - self.m, 1e-6))
+            factor = 1.0 - lam * (x - self.m)
+        else:
+            lam = min(lam, self.truncation / max(self.m, 1e-6))
+            factor = 1.0 + lam * (x - self.m)
+        self.log_e += math.log(max(factor, 1e-12))
+        self.t = t
+        self._sum_x += x
+        self._mu_hat = (0.5 + self._sum_x) / (t + 1)
+        self._var_acc += (x - self._mu_hat) ** 2
+        self._sigma2_hat = self._var_acc / (t + 1)
+
+    @property
+    def e_value(self) -> float:
+        return math.exp(min(self.log_e, 700.0))
+
+    def p_analogue(self) -> float:
+        """Anytime-valid p-value analogue: min(1, 1/e)."""
+        return min(1.0, 1.0 / max(self.e_value, 1e-300))
+
+
+class PairedSequentialTest:
+    """Three-outcome anytime-valid test on paired differences d in [-1, 1].
+
+    Delta = E[d] (candidate minus baseline). Regression margin `delta` in
+    absolute metric points: the H0 boundary sits at Delta = -delta.
+    """
+
+    def __init__(self, delta: float = 0.005, alpha: float = 0.05,
+                 alpha_pass: Optional[float] = None):
+        self.delta = float(delta)
+        self.alpha = float(alpha)
+        self.alpha_pass = float(alpha_pass if alpha_pass is not None else alpha)
+        m = (1.0 - self.delta) / 2.0  # Delta = -delta  <=>  mu = (1-delta)/2
+        self.cs = EmpiricalBernsteinCS(alpha=self.alpha)
+        self.e_reg = OneSidedEProcess(m, side="below", alpha_ref=self.alpha)
+        self.e_pass = OneSidedEProcess(m, side="above", alpha_ref=self.alpha_pass)
+        # object-level accounting (for reporting + Bayesian complement)
+        self.n_objects = 0
+        self.n_clusters = 0
+        self.sum_baseline = 0.0
+        self.sum_candidate = 0.0
+        self.n01 = 0  # baseline correct, candidate wrong  (regression pairs)
+        self.n10 = 0  # candidate correct, baseline wrong  (improvement pairs)
+        self.decision = DECISION_INSUFFICIENT
+        self.decided_at_n: Optional[int] = None
+        self.trajectory: List[Dict] = []
+
+    # ---- updates -------------------------------------------------------
+
+    def update_clusters(self, cluster_means) -> None:
+        """Feed per-cluster mean paired differences (each in [-1, 1])."""
+        for d in cluster_means:
+            x = (float(d) + 1.0) / 2.0
+            self.cs.update(x)
+            self.e_reg.update(x)
+            self.e_pass.update(x)
+            self.n_clusters += 1
+
+    def record_objects(self, b, c) -> None:
+        """Track object-level paired outcomes for counts and the posterior."""
+        import numpy as np
+        b = np.asarray(b, dtype=bool)
+        c = np.asarray(c, dtype=bool)
+        self.n_objects += int(b.size)
+        self.sum_baseline += float(b.sum())
+        self.sum_candidate += float(c.sum())
+        self.n01 += int(np.sum(b & ~c))
+        self.n10 += int(np.sum(~b & c))
+
+    # ---- state ---------------------------------------------------------
+
+    def delta_interval(self) -> tuple:
+        lo, hi = self.cs.interval()
+        return (2.0 * lo - 1.0, 2.0 * hi - 1.0)
+
+    def delta_estimate(self) -> Optional[float]:
+        if self.n_objects == 0:
+            return None
+        return (self.sum_candidate - self.sum_baseline) / self.n_objects
+
+    def bayes_p_regression(self) -> Optional[float]:
+        """Beta-Binomial posterior P(Delta < -delta | data).
+
+        McNemar decomposition: among D = n01 + n10 discordant pairs let
+        theta = P(pair is a 01 regression pair). With plug-in discordance rate
+        nu = D/n, Delta = nu * (1 - 2*theta), so
+            Delta < -delta  <=>  theta > (1 + delta*n/D) / 2.
+        Prior theta ~ Beta(1,1).
+        """
+        d_total = self.n01 + self.n10
+        if self.n_objects == 0 or d_total == 0:
+            return None
+        threshold = (1.0 + self.delta * self.n_objects / d_total) / 2.0
+        if threshold >= 1.0:
+            return 0.0
+        return float(_sstats.beta.sf(threshold, 1 + self.n01, 1 + self.n10))
+
+    def evaluate(self, alpha_reg: Optional[float] = None,
+                 alpha_pass: Optional[float] = None) -> str:
+        """Update the sticky three-outcome decision at the given levels."""
+        if self.decision != DECISION_INSUFFICIENT:
+            return self.decision
+        a_r = alpha_reg if alpha_reg is not None else self.alpha
+        a_p = alpha_pass if alpha_pass is not None else self.alpha_pass
+        if self.e_reg.e_value >= 1.0 / a_r:
+            self.decision = DECISION_REGRESSION
+            self.decided_at_n = self.n_objects
+        elif self.e_pass.e_value >= 1.0 / a_p:
+            self.decision = DECISION_PASS
+            self.decided_at_n = self.n_objects
+        return self.decision
+
+    def snapshot(self) -> Dict:
+        lo, hi = self.delta_interval()
+        est = self.delta_estimate()
+        return {
+            "n": self.n_objects,
+            "n_clusters": self.n_clusters,
+            "delta_estimate": None if est is None else round(est, 5),
+            "delta_ci": [round(lo, 5), round(hi, 5)],
+            "e_regression": round(self.e_reg.e_value, 4),
+            "e_pass": round(self.e_pass.e_value, 4),
+            "p_analogue": round(self.e_reg.p_analogue(), 6),
+            "bayes_p_regression": (None if self.bayes_p_regression() is None
+                                   else round(self.bayes_p_regression(), 5)),
+            "decision": self.decision,
+        }
+
+    def record_trajectory_point(self) -> None:
+        """Append an evidence-vs-samples point (for the dashboard chart)."""
+        self.trajectory.append({
+            "n": self.n_objects,
+            "n_clusters": self.n_clusters,
+            "delta_estimate": self.delta_estimate(),
+            "delta_lower": round(self.delta_interval()[0], 5),
+            "delta_upper": round(self.delta_interval()[1], 5),
+            "log_e_regression": round(self.e_reg.log_e, 4),
+            "log_e_pass": round(self.e_pass.log_e, 4),
+            "decision": self.decision,
+        })
+
+
+def approx_mde(var_d: float, n_eff: float, delta: float, alpha: float,
+               power: float = 0.9, anytime_inflation: float = 1.4) -> Optional[float]:
+    """Approximate minimum detectable absolute effect at the current n_eff.
+
+    Fixed-n normal approximation (z_{1-a} + z_{power}) * sqrt(var/n_eff),
+    inflated for the anytime-valid overhead, plus the practical margin delta
+    (effects must exceed the margin to be declarable)."""
+    if n_eff <= 1 or var_d <= 0:
+        return None
+    z = _sstats.norm.ppf(1 - alpha) + _sstats.norm.ppf(power)
+    return float(delta + anytime_inflation * z * math.sqrt(var_d / n_eff))

sensorflow/seqeval/units.py

@@ -0,0 +1,107 @@
+"""Statistical-unit handling: clustering, intra-cluster correlation, design effect.
+
+Frames/objects are NOT independent observations: objects in the same container
+(scene/track) share weather, lighting, scenario, scene-level "hardness", and the
+same model failure modes. Treating them as iid understates variance and makes
+any test anti-conservative.
+
+The seqeval convention, used by every module downstream:
+
+  * The inference unit is the CLUSTER: all sampled objects of one stratum that
+    live in the same container. Per-cluster mean paired differences (bounded in
+    [-1, 1]) are what feed the confidence sequences and e-processes. This is
+    the clustered analogue of a paired analysis and is valid regardless of the
+    within-cluster correlation structure.
+  * Object-level counts are still tracked for reporting; the *effective* sample
+    size n_eff = n / DEFF with DEFF = 1 + (m_bar - 1) * ICC is attached to every
+    evidence record so readers can see how much clustering cost us.
+"""
+
+from __future__ import annotations
+
+from typing import Dict, List, Tuple
+
+import numpy as np
+
+
+def cluster_units(values: np.ndarray, cluster_ids: np.ndarray,
+                  weights: np.ndarray | None = None) -> Tuple[np.ndarray, np.ndarray]:
+    """Collapse object-level values into per-cluster (weight-normalized) means.
+
+    Returns (cluster_means, cluster_sizes). Means are convex combinations of the
+    inputs, so they stay inside the input range — a requirement for the bounded
+    confidence sequences downstream.
+    """
+    values = np.asarray(values, dtype=np.float64)
+    cluster_ids = np.asarray(cluster_ids)
+    if weights is None:
+        weights = np.ones_like(values)
+    else:
+        weights = np.asarray(weights, dtype=np.float64)
+    order = np.argsort(cluster_ids, kind="stable")
+    cid, v, w = cluster_ids[order], values[order], weights[order]
+    uniq, start = np.unique(cid, return_index=True)
+    bounds = np.append(start, cid.size)
+    wsum = np.add.reduceat(w, start)
+    vsum = np.add.reduceat(v * w, start)
+    sizes = np.diff(bounds)
+    means = vsum / np.where(wsum > 0, wsum, 1.0)
+    return means, sizes
+
+
+def icc_anova(values: np.ndarray, cluster_ids: np.ndarray) -> float:
+    """One-way ANOVA (method-of-moments) estimator of the intra-cluster
+    correlation. Clipped to [0, 1]; returns 0.0 when it cannot be estimated."""
+    values = np.asarray(values, dtype=np.float64)
+    cluster_ids = np.asarray(cluster_ids)
+    order = np.argsort(cluster_ids, kind="stable")
+    v, cid = values[order], cluster_ids[order]
+    uniq, start = np.unique(cid, return_index=True)
+    k = uniq.size
+    n = v.size
+    if k < 2 or n <= k:
+        return 0.0
+    sizes = np.diff(np.append(start, n)).astype(np.float64)
+    sums = np.add.reduceat(v, start)
+    means = sums / sizes
+    grand = v.mean()
+    ss_between = float(np.sum(sizes * (means - grand) ** 2))
+    ss_within = float(np.sum((v - np.repeat(means, sizes.astype(int))) ** 2))
+    ms_between = ss_between / (k - 1)
+    ms_within = ss_within / (n - k)
+    # average cluster size adjusted for imbalance (n0 in Donner & Klar)
+    n0 = (n - float(np.sum(sizes ** 2)) / n) / (k - 1)
+    denom = ms_between + (n0 - 1) * ms_within
+    if denom <= 0:
+        return 0.0
+    return float(np.clip((ms_between - ms_within) / denom, 0.0, 1.0))
+
+
+def design_effect(cluster_sizes: np.ndarray, icc: float) -> float:
+    """Kish design effect DEFF = 1 + (m_bar - 1) * ICC, with m_bar the
+    size-weighted mean cluster size (sum m^2 / sum m)."""
+    sizes = np.asarray(cluster_sizes, dtype=np.float64)
+    if sizes.size == 0 or sizes.sum() <= 0:
+        return 1.0
+    m_bar = float(np.sum(sizes ** 2) / np.sum(sizes))
+    return max(1.0, 1.0 + (m_bar - 1.0) * icc)
+
+
+def effective_sample_size(n: int, deff: float) -> float:
+    return float(n) / max(deff, 1.0)
+
+
+def cluster_summary(values: np.ndarray, cluster_ids: np.ndarray) -> Dict:
+    """Full clustering diagnostics for one node/stratum (for the ledger)."""
+    values = np.asarray(values, dtype=np.float64)
+    _, sizes = cluster_units(values, cluster_ids)
+    icc = icc_anova(values, cluster_ids)
+    deff = design_effect(sizes, icc)
+    return {
+        "n": int(values.size),
+        "n_clusters": int(sizes.size),
+        "mean_cluster_size": round(float(sizes.mean()), 3) if sizes.size else 0.0,
+        "icc": round(icc, 4),
+        "design_effect": round(deff, 3),
+        "n_effective": round(effective_sample_size(values.size, deff), 1),
+    }

tests/test_safety/conftest.py

@@ -0,0 +1,80 @@
+"""Shared fixtures for the safety layer test suite.
+
+- mega_env: isolated megaeval root with a small population and two published
+  runs (a good candidate and a deliberately regressed one) — mirrors
+  tests/test_megaeval/conftest.py.
+- safety_root: session-scoped isolated runs/safety replacement.
+- fresh_safety_root: function-scoped pristine safety root (for tests that need
+  an empty scenario DB / no cached CSI / no supplements).
+- eval_env: fresh labeleval store with a small synthetic dataset + labels.
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from sensorflow.megaeval import population as pop_mod
+from sensorflow.megaeval.runs import get_mega_store, reset_mega_store
+from sensorflow.safety import scenario_db
+from sensorflow.safety import store as safety_store
+
+
+@pytest.fixture(scope="session")
+def mega_env(tmp_path_factory):
+    root = tmp_path_factory.mktemp("megaeval-safety")
+    pop_mod.set_mega_root(str(root))
+    reset_mega_store()
+    meta = pop_mod.generate_population("safety-pop", num_objects=8_000, seed=17)
+    store = get_mega_store()
+
+    good = store.create_run(population_id=meta["population_id"],
+                            model_version="model-v41", worker_delay_s=0.0)
+    store.execute_sync(good)
+    assert good.status == "published", good.error
+
+    bad = store.create_run(population_id=meta["population_id"],
+                           model_version="model-v42-regressed",
+                           overrides={"night_penalty": 0.35, "vru_penalty": 0.12},
+                           worker_delay_s=0.0)
+    store.execute_sync(bad)
+    assert bad.status == "published", bad.error
+
+    yield {"root": root, "meta": meta, "store": store, "good": good, "bad": bad}
+
+    pop_mod.set_mega_root("runs/megaeval")
+    reset_mega_store()
+
+
+@pytest.fixture(scope="session")
+def safety_root(tmp_path_factory):
+    root = tmp_path_factory.mktemp("safety-root")
+    safety_store.set_safety_root(str(root))
+    scenario_db.reset_db()
+    yield root
+    safety_store.set_safety_root(os.path.join("runs", "safety"))
+    scenario_db.reset_db()
+
+
+@pytest.fixture()
+def fresh_safety_root(safety_root, tmp_path):
+    """Pristine safety root for one test; restores the session root after."""
+    fresh = tmp_path / "safety"
+    safety_store.set_safety_root(str(fresh))
+    scenario_db.reset_db()
+    yield fresh
+    safety_store.set_safety_root(str(safety_root))
+    scenario_db.reset_db()
+
+
+@pytest.fixture()
+def eval_env(tmp_path):
+    from sensorflow.evaluation import synthetic
+    from sensorflow.evaluation.records import reset_store
+
+    store = reset_store(tmp_path)
+    ds = synthetic.generate_dataset(store, num_sequences=2, frames_per_sequence=8,
+                                    seed=11)
+    synthetic.generate_labels(store, ds)
+    return store, ds

tests/test_safety/test_api_safety.py

@@ -0,0 +1,203 @@
+"""Live verification of every /api/safety/* endpoint through the real app
+(in-process fastapi TestClient against app_backend.app — no running servers
+are touched)."""
+
+from __future__ import annotations
+
+import pytest
+from fastapi.testclient import TestClient
+
+LENIENT = {
+    "scenario_quality": {"min_geometric_pass_rate": 0.05},
+    "coverage": {"min_coverage_rate": 0.0, "min_production_weighted_coverage": 0.0,
+                 "min_samples": 5, "max_ci_width": 0.99},
+}
+
+
+@pytest.fixture(scope="module")
+def client():
+    from app_backend import app
+    return TestClient(app)
+
+
+@pytest.fixture()
+def api_env(mega_env, fresh_safety_root, tmp_path):
+    """Isolated safety root + labeleval store with one dataset, plus mega runs."""
+    from sensorflow.evaluation import synthetic
+    from sensorflow.evaluation.records import reset_store
+
+    store = reset_store(tmp_path / "eval")
+    ds = synthetic.generate_dataset(store, num_sequences=2, frames_per_sequence=6,
+                                    seed=23)
+    synthetic.generate_labels(store, ds)
+    return {"mega": mega_env, "store": store, "ds": ds}
+
+
+def test_odd_endpoints(client, api_env):
+    assert client.get("/api/safety/odd/taxonomy").status_code == 200
+
+    run_id = api_env["mega"]["good"].run_id
+    r = client.get("/api/safety/odd/coverage",
+                   params={"run": run_id, "dims": "weather,lighting",
+                           "min_samples": 10_000})
+    assert r.status_code == 200
+    cov = r.json()
+    assert cov["run_id"] == run_id and cov["gaps"]
+
+    gap = cov["gaps"][0]
+    r = client.post("/api/safety/odd/fill-gap",
+                    json={"run": run_id, "cell": gap["cell"], "num_sequences": 1,
+                          "frames_per_sequence": 10})
+    assert r.status_code == 200
+    assert r.json()["objects_added"] > 0
+
+    assert client.get("/api/safety/odd/coverage",
+                      params={"run": "eval-nope"}).status_code == 404
+
+
+def test_gates_and_evidence_endpoints(client, api_env):
+    good = api_env["mega"]["good"].run_id
+    bad = api_env["mega"]["bad"].run_id
+
+    r = client.get("/api/safety/gates/policy")
+    assert r.status_code == 200 and "coverage" in r.json()["policy"]
+
+    r = client.post("/api/safety/gates/policy",
+                    json={"overrides": {"safety": {"max_csi_increase_ratio": 0.05}}})
+    assert r.status_code == 200
+    assert r.json()["policy"]["safety"]["max_csi_increase_ratio"] == 0.05
+
+    r = client.post("/api/safety/gates/evaluate",
+                    json={"candidate_run": bad, "baseline_run": good,
+                          "policy_overrides": LENIENT})
+    assert r.status_code == 200
+    res = r.json()
+    assert res["decision"] == "BLOCKED"
+    assert "regression" in res["blocking_gates"]
+
+    r = client.get(f"/api/safety/gates/result/{bad}")
+    assert r.status_code == 200 and r.json()["decision"] == "BLOCKED"
+
+    r = client.get(f"/api/safety/evidence/{bad}")
+    assert r.status_code == 200
+    assert r.json()["package_id"] == f"sep-{bad}"
+
+    r = client.get(f"/api/safety/evidence/{bad}", params={"format": "markdown"})
+    assert r.status_code == 200
+    assert "# Safety Evidence Package" in r.json()["markdown"]
+
+    assert client.get("/api/safety/evidence/eval-none").status_code == 404
+
+
+def test_ssam_endpoints(client, api_env):
+    r = client.post("/api/safety/ssam/analyze",
+                    json={"scenario": "crossing", "seed": 0, "reaction_delay_s": 0.9})
+    assert r.status_code == 200
+    body = r.json()
+    assert body["aggregate"]["num_conflicts"] >= 1
+    assert body["generated"]["simulated"] is True
+
+    # explicit trajectories path
+    states = [{"t": round(i * 0.1, 3), "x": 10.0 * i * 0.1, "y": 0.0,
+               "speed": 10.0, "heading": 0.0} for i in range(21)]
+    still = [{"t": round(i * 0.1, 3), "x": 30.0, "y": 0.0, "speed": 0.0,
+              "heading": 0.0} for i in range(21)]
+    r = client.post("/api/safety/ssam/analyze", json={"trajectories": [
+        {"vehicle_id": "A", "states": states},
+        {"vehicle_id": "B", "states": still}]})
+    assert r.status_code == 200
+    assert r.json()["aggregate"]["num_conflicts"] == 1
+
+    run_id = api_env["mega"]["good"].run_id
+    r = client.get("/api/safety/ssam/summary", params={"run": run_id})
+    assert r.status_code == 200
+    assert r.json()["aggregate_csi"] > 0
+
+
+def test_calibration_endpoints(client, api_env):
+    r = client.post("/api/safety/calibration/validate",
+                    json={"mode": "miscalibrated", "rotation_offset_deg": 2.0})
+    assert r.status_code == 200
+    assert r.json()["status"] == "MISCALIBRATED"
+
+    r = client.get("/api/safety/calibration/status")
+    assert r.status_code == 200 and r.json()["status"] == "MISCALIBRATED"
+
+    r = client.post("/api/safety/calibration/validate", json={"mode": "clean"})
+    assert r.json()["status"] == "CALIBRATED"
+
+
+def test_discrepancy_endpoints(client, api_env):
+    ds_id = api_env["ds"].dataset_id
+    r = client.post("/api/safety/discrepancy/mine",
+                    json={"dataset_id": ds_id,
+                          "profile": {"extra_miss_rate": 0.5}})
+    assert r.status_code == 200
+    report = r.json()
+    assert report["totals"]["discrepancies"] > 0
+
+    r = client.get("/api/safety/discrepancy/summary", params={"dataset_id": ds_id})
+    assert r.status_code == 200
+    assert r.json()["dataset_id"] == ds_id
+
+    r = client.get("/api/safety/discrepancy/summary")
+    assert r.status_code == 200 and len(r.json()["datasets"]) == 1
+
+
+def test_scenario_endpoints(client, api_env):
+    ds_id = api_env["ds"].dataset_id
+    # seed content through discrepancy mining
+    client.post("/api/safety/discrepancy/mine",
+                json={"dataset_id": ds_id, "profile": {"extra_miss_rate": 0.5}})
+
+    r = client.get("/api/safety/scenarios", params={"source": "discrepancy"})
+    assert r.status_code == 200
+    body = r.json()
+    assert body["counts"]["total"] > 0
+    assert all(s["source"] == "discrepancy" for s in body["scenarios"])
+
+    r = client.post("/api/safety/scenarios/populate")
+    assert r.status_code == 200
+
+    r = client.post("/api/safety/scenarios/export", json={"source": "discrepancy"})
+    assert r.status_code == 200
+    bundle = r.json()
+    assert bundle["bundle_format"] == "sensorflow-scenario-bundle/v1"
+    assert bundle["count"] == len(bundle["scenarios"]) > 0
+
+
+def test_semantic_search_endpoint(client, api_env):
+    r = client.post("/api/safety/semantic-search",
+                    json={"concept": "hesitant vulnerable road user near lane edge",
+                          "target": "containers", "k": 8, "use_llm": False})
+    assert r.status_code == 200
+    body = r.json()
+    assert body["results"]
+    assert body["stage2"]["provider"] == "offline_deterministic"
+    assert body["results"][0]["explanations"]["stage2_reasoning"]
+
+    r = client.post("/api/safety/semantic-search",
+                    json={"concept": "night rain", "target": "containers",
+                          "filters": {"lighting": "night"}, "use_llm": False})
+    assert all(x["lighting"] == "night" for x in r.json()["results"])
+
+
+def test_consensus_endpoint(client, api_env):
+    store, ds = api_env["store"], api_env["ds"]
+    from sensorflow.evaluation.graders import grade_annotation
+    frames = {f.frame_id: f for f in store.where("frames", dataset_id=ds.dataset_id)}
+    for ann in [a for a in store.where("annotations", dataset_id=ds.dataset_id)
+                if a.bbox_3d][:6]:
+        grade_annotation(store, ann, frames[ann.frame_id])
+
+    r = client.get("/api/safety/consensus/summary",
+                   params={"dataset_id": ds.dataset_id, "examples": 3})
+    assert r.status_code == 200
+    body = r.json()
+    assert "kendall_tau_confidence_vs_consensus" in body["statistics"]
+    assert body["examples"]
+    ex = body["examples"][0]
+    assert "consensus_score_vector" in ex and "mbr" in ex
+
+    assert client.get("/api/safety/consensus/summary",
+                      params={"dataset_id": "ds-none"}).status_code == 404

tests/test_safety/test_calibration.py

@@ -0,0 +1,65 @@
+"""Multi-sensor calibration validation: clean scenes pass, an injected 2-degree
+extrinsic rotation must be flagged as MISCALIBRATED (systematic bias), tampered
+2D boxes must be flagged as PERCEPTION_FAILURE (random disagreement) mapped to
+the platform's SENSOR_DISAGREEMENT failure reason."""
+
+from __future__ import annotations
+
+import pytest
+
+from sensorflow.safety import calibration as cal
+
+
+def test_clean_scene_passes(fresh_safety_root):
+    res = cal.run_validation(mode="clean", seed=3)
+    assert res["status"] == "CALIBRATED"
+    assert res["passed"] is True
+    assert all(c["passed"] for c in res["checks"])
+    assert not any(o["flagged"] for o in res["per_object"])
+    assert res["metrics"]["bias_px"] <= res["thresholds"]["bias_px"]
+    assert res["simulated"] is True
+
+
+def test_two_degree_rotation_flagged_as_miscalibration(fresh_safety_root):
+    res = cal.run_validation(mode="miscalibrated", rotation_offset_deg=2.0, seed=3)
+    assert res["status"] == "MISCALIBRATED"
+    assert res["passed"] is False
+    # systematic: every object shares the bias
+    assert all(o["flagged"] for o in res["per_object"])
+    assert all(o["failure_reason"] == "MISCALIBRATION" for o in res["per_object"])
+    # recovered rotation magnitude is in the right ballpark
+    assert res["metrics"]["estimated_rotation_offset_deg"] == pytest.approx(2.0, abs=0.8)
+    bias_check = next(c for c in res["checks"] if c["name"] == "systematic_bias_px")
+    assert not bias_check["passed"]
+
+
+def test_perception_failure_distinguished_from_miscalibration(fresh_safety_root):
+    res = cal.run_validation(mode="perception_failure", tamper_fraction=0.3, seed=3)
+    assert res["status"] == "PERCEPTION_FAILURE"
+    flagged = [o for o in res["per_object"] if o["flagged"]]
+    assert flagged and len(flagged) < len(res["per_object"])  # minority, not systematic
+    # maps onto the existing platform failure reason
+    assert all(o["failure_reason"] == "SENSOR_DISAGREEMENT" for o in flagged)
+
+
+def test_status_persisted_and_alert_emitted(fresh_safety_root, tmp_path):
+    from sensorflow.evaluation.records import reset_store
+    store = reset_store(tmp_path / "eval")
+
+    assert cal.latest_status() is None
+    cal.run_validation(mode="miscalibrated", rotation_offset_deg=2.0, seed=3)
+    status = cal.latest_status()
+    assert status is not None and status["status"] == "MISCALIBRATED"
+    # failure surfaced in the existing labeleval alert store
+    alerts = [a for a in store.all("alerts") if a.kind == "calibration"]
+    assert alerts and alerts[0].severity == "critical"
+
+
+def test_extrinsic_projection_roundtrip():
+    import numpy as np
+    ext = cal.make_extrinsic()
+    scene = cal.generate_scene(num_objects=6, seed=5)
+    for obj in scene["objects"]:
+        uv = cal.project_points(np.asarray(obj["points_lidar"]), ext)
+        assert uv.shape[1] == 2
+        assert np.isfinite(uv).all()

tests/test_safety/test_consensus.py

@@ -0,0 +1,95 @@
+"""Additive consensus extensions: consensus score vector, Kendall's tau, and
+Minimum Bayes Risk selection — exact values on small fixtures, plus wiring into
+the grader comparison records."""
+
+from __future__ import annotations
+
+import pytest
+
+from sensorflow.evaluation.graders import (
+    consensus_score_vector,
+    dataset_grader_statistics,
+    grade_annotation,
+    kendalls_tau,
+    mbr_consensus_select,
+    mbr_utility,
+)
+
+
+def test_kendalls_tau_exact():
+    assert kendalls_tau([1, 2, 3], [1, 2, 3]) == (1.0, pytest.approx(0.333333, abs=1e-3))
+    tau, _ = kendalls_tau([1, 2, 3], [3, 2, 1])
+    assert tau == -1.0
+    # one swapped pair of three: tau-b = (2-1)/3 = 1/3
+    tau, _ = kendalls_tau([1, 2, 3], [1, 3, 2])
+    assert tau == pytest.approx(1 / 3, abs=1e-4)
+    # degenerate (constant) input handled
+    assert kendalls_tau([1, 1], [2, 2]) == (0.0, 1.0)
+
+
+def test_consensus_score_vector_exact():
+    votes = {"a": "vehicle", "b": "vehicle", "c": "vehicle", "d": "vehicle",
+             "e": "truck"}
+    detail = {"iou": 0.5, "center_distance": 1.0, "dimension_diff": 0.2,
+              "orientation_diff_deg": 45.0}
+    v = consensus_score_vector(votes, detail, temporal=1.0)
+    assert v["class_majority_share"] == pytest.approx(0.8)
+    assert v["class_weighted_agreement"] == pytest.approx(0.8)  # uniform weights
+    assert v["spatial_iou"] == pytest.approx(0.5)
+    assert v["spatial_center"] == pytest.approx(0.5)       # 1 - 1.0/2.0
+    assert v["spatial_dims"] == pytest.approx(0.8)         # 1 - 0.2
+    assert v["spatial_orientation"] == pytest.approx(0.75)  # 1 - 45/180
+    assert v["temporal"] == 1.0
+
+
+BOX = [0.0, 0.0, 0.8, 4.5, 1.9, 1.6, 0.0]
+FAR_BOX = [50.0, 50.0, 0.8, 4.5, 1.9, 1.6, 0.0]
+
+
+def test_mbr_utility_exact():
+    a = {"class_name": "vehicle", "bbox_3d": BOX}
+    b = {"class_name": "vehicle", "bbox_3d": list(BOX)}
+    c = {"class_name": "pedestrian", "bbox_3d": FAR_BOX}
+    assert mbr_utility(a, b) == pytest.approx(1.0)   # class match + IoU 1
+    assert mbr_utility(a, c) == pytest.approx(0.0)   # no match, no overlap
+    d = {"class_name": "vehicle", "bbox_3d": FAR_BOX}
+    assert mbr_utility(a, d) == pytest.approx(0.5)   # class only
+
+
+def test_mbr_selects_minimum_expected_risk():
+    candidates = [
+        {"source": "auto_label", "class_name": "vehicle", "bbox_3d": BOX},
+        {"source": "vendor_gt", "class_name": "vehicle", "bbox_3d": list(BOX)},
+        {"source": "vlm_label", "class_name": "pedestrian", "bbox_3d": FAR_BOX},
+    ]
+    res = mbr_consensus_select(candidates)
+    # risks: agreeing pair 1 - mean(1, 0) = 0.5 each; outlier 1 - mean(0, 0) = 1
+    risks = [c["expected_risk"] for c in res["candidates"]]
+    assert risks == [0.5, 0.5, 1.0]
+    assert res["selected_index"] == 0  # first of the tied minimum
+    assert res["selected"]["source"] == "auto_label"
+
+    with pytest.raises(ValueError):
+        mbr_consensus_select([])
+
+
+def test_grade_annotation_carries_extension_evidence(eval_env):
+    store, ds = eval_env
+    frames = {f.frame_id: f for f in store.where("frames", dataset_id=ds.dataset_id)}
+    anns = [a for a in store.where("annotations", dataset_id=ds.dataset_id)
+            if a.bbox_3d][:6]
+    assert len(anns) >= 2
+    for ann in anns:
+        cmp = grade_annotation(store, ann, frames[ann.frame_id])
+        vector_keys = {k for k in cmp.kappa_stats if k.startswith("csv_")}
+        assert {"csv_class_majority_share", "csv_spatial_iou",
+                "csv_temporal"} <= vector_keys
+        assert "mbr_selected_idx" in cmp.kappa_stats
+        assert "mbr_risk_auto_label" in cmp.kappa_stats
+        # canonical scalar consensus still present and bounded
+        assert cmp.consensus is not None and 0.0 <= cmp.consensus <= 1.0
+
+    stats = dataset_grader_statistics(store, ds.dataset_id)
+    assert "cohen_kappa" in stats
+    assert "kendall_tau_confidence_vs_consensus" in stats
+    assert -1.0 <= stats["kendall_tau_confidence_vs_consensus"] <= 1.0

tests/test_safety/test_discrepancy.py

@@ -0,0 +1,83 @@
+"""Auto-label discrepancy mining: injected online misses are found, cohort
+rates are consistent, and findings feed the rare-event store + scenario DB."""
+
+from __future__ import annotations
+
+import pytest
+
+from sensorflow.safety import discrepancy
+from sensorflow.safety.scenario_db import get_db
+
+AGGRESSIVE = {"extra_miss_rate": 0.5, "class_confusion_rate": 0.2,
+              "position_noise_m": 1.2}
+
+
+def test_mine_finds_injected_online_misses(eval_env, fresh_safety_root):
+    store, ds = eval_env
+    report = discrepancy.mine(store, ds.dataset_id, profile=AGGRESSIVE, seed=5)
+
+    t = report["totals"]
+    assert t["objects"] > 0
+    assert t["discrepancies"] > 0
+    assert t["discrepancy_rate"] == pytest.approx(
+        t["discrepancies"] / t["objects"], abs=1e-3)
+    # a 50% extra miss rate must produce MISSED_ONLINE findings
+    assert report["by_type"].get("MISSED_ONLINE", 0) > 0
+    assert sum(report["by_type"].values()) == t["discrepancies"]
+    assert report["simulated"] is True
+
+    # cohort accounting: rates in [0,1], hits never exceed objects
+    for c in report["cohorts"]:
+        assert 0.0 <= c["discrepancy_rate"] <= 1.0
+        assert c["discrepancies"] <= c["objects"]
+    assert sum(c["objects"] for c in report["cohorts"]) == t["objects"]
+
+    # every discrepancy is a queryable scenario-mining record
+    for d in report["discrepancies"][:10]:
+        assert {"discrepancy_id", "frame_id", "gt_id", "type", "class_name",
+                "weather", "time_of_day", "severity"} <= set(d)
+
+
+def test_mine_is_deterministic(eval_env, fresh_safety_root):
+    store, ds = eval_env
+    r1 = discrepancy.mine(store, ds.dataset_id, profile=AGGRESSIVE, seed=5,
+                          persist=False)
+    r2 = discrepancy.mine(store, ds.dataset_id, profile=AGGRESSIVE, seed=5,
+                          persist=False)
+    assert r1["by_type"] == r2["by_type"]
+    assert [d["gt_id"] for d in r1["discrepancies"]] == \
+           [d["gt_id"] for d in r2["discrepancies"]]
+
+
+def test_mine_feeds_rare_events_and_scenario_db(eval_env, fresh_safety_root):
+    store, ds = eval_env
+    before_events = len(store.all("rare_events"))
+    report = discrepancy.mine(store, ds.dataset_id, profile=AGGRESSIVE, seed=5)
+
+    critical = [d for d in report["discrepancies"] if d["severity"] == "critical"]
+    if critical:  # pedestrian/cyclist online misses
+        mined_events = [e for e in store.all("rare_events")
+                        if e.scenario_type == "online_perception_miss"]
+        assert len(store.all("rare_events")) > before_events
+        assert mined_events and all(e.severity == "critical" for e in mined_events)
+
+    recs = get_db().search(source="discrepancy", limit=1000)
+    high = [d for d in report["discrepancies"] if d["severity"] in ("high", "critical")]
+    assert len(recs) == len({(d["gt_id"], d["type"]) for d in high})
+
+
+def test_latest_summary_roundtrip(eval_env, fresh_safety_root):
+    store, ds = eval_env
+    assert discrepancy.latest_summary(ds.dataset_id) is None
+    discrepancy.mine(store, ds.dataset_id, profile=AGGRESSIVE, seed=5)
+    summary = discrepancy.latest_summary(ds.dataset_id)
+    assert summary is not None
+    assert summary["dataset_id"] == ds.dataset_id
+    assert "discrepancies" not in summary  # slimmed
+    assert len(summary["sample_discrepancies"]) <= 25
+
+
+def test_mine_unknown_dataset_raises(eval_env, fresh_safety_root):
+    store, _ = eval_env
+    with pytest.raises(KeyError):
+        discrepancy.mine(store, "ds-nonexistent")

tests/test_safety/test_gates.py

@@ -0,0 +1,124 @@
+"""Layered release gating: policy handling, blocking behavior with the right
+evidence, and the Safety Evidence Package (JSON + markdown)."""
+
+from __future__ import annotations
+
+import pytest
+
+from sensorflow.safety import gates
+
+LENIENT = {
+    "scenario_quality": {"min_geometric_pass_rate": 0.05},
+    "coverage": {"min_coverage_rate": 0.0, "min_production_weighted_coverage": 0.0,
+                 "min_samples": 5, "max_ci_width": 0.99},
+}
+
+
+@pytest.fixture()
+def eval_store(tmp_path):
+    from sensorflow.evaluation.records import reset_store
+    return reset_store(tmp_path / "eval")
+
+
+def test_policy_defaults_merge_and_persist(fresh_safety_root):
+    p = gates.get_policy()
+    assert p == gates.DEFAULT_POLICY
+    updated = gates.set_policy({"safety": {"max_csi_increase_ratio": 0.5}})
+    assert updated["safety"]["max_csi_increase_ratio"] == 0.5
+    # untouched sections keep defaults
+    assert updated["coverage"]["min_samples"] == gates.DEFAULT_POLICY["coverage"]["min_samples"]
+    assert gates.get_policy()["safety"]["max_csi_increase_ratio"] == 0.5
+
+
+def test_all_gates_pass_for_good_candidate(mega_env, fresh_safety_root, eval_store):
+    res = gates.evaluate_gates(mega_env["store"], mega_env["good"], mega_env["good"],
+                               policy_overrides=LENIENT)
+    assert res["decision"] == "RELEASE_READY"
+    assert res["blocking_gates"] == []
+    by_gate = {g["gate"]: g for g in res["gates"]}
+    assert set(by_gate) == {"scenario_quality", "coverage", "regression",
+                            "safety", "release_readiness"}
+    assert all(g["status"] == "PASS" for g in res["gates"])
+    # candidate == baseline -> zero CSI delta, PROMOTE recommendation
+    assert by_gate["safety"]["evidence"]["csi_increase_ratio"] == pytest.approx(0.0)
+    assert by_gate["regression"]["evidence"]["blockers"] == []
+    # every gate carries checks with thresholds and actuals
+    for g in res["gates"]:
+        for c in g["checks"]:
+            assert {"check", "actual", "threshold", "passed"} <= set(c)
+
+
+def test_regressed_candidate_blocks_regression_and_safety(mega_env, fresh_safety_root,
+                                                          eval_store):
+    overrides = {**LENIENT, "safety": {"max_csi_increase_ratio": 0.05}}
+    res = gates.evaluate_gates(mega_env["store"], mega_env["bad"], mega_env["good"],
+                               policy_overrides=overrides)
+    assert res["decision"] == "BLOCKED"
+    assert "regression" in res["blocking_gates"]
+    assert "safety" in res["blocking_gates"]
+    by_gate = {g["gate"]: g for g in res["gates"]}
+    # regression gate carries the megaeval blockers as evidence
+    assert by_gate["regression"]["evidence"]["blockers"]
+    # safety gate shows candidate CSI above baseline
+    ev = by_gate["safety"]["evidence"]
+    assert ev["candidate_csi"] > ev["baseline_csi"]
+    # readiness gate blocked and names the first blocking gate
+    rr = by_gate["release_readiness"]
+    assert rr["status"] == "BLOCK"
+    assert rr["evidence"]["first_blocking_gate"] in res["blocking_gates"]
+
+
+def test_coverage_gate_blocks_on_impossible_threshold(mega_env, fresh_safety_root,
+                                                      eval_store):
+    overrides = {**LENIENT,
+                 "coverage": {"min_coverage_rate": 1.01,
+                              "min_production_weighted_coverage": 0.0}}
+    res = gates.evaluate_gates(mega_env["store"], mega_env["good"], mega_env["good"],
+                               policy_overrides=overrides)
+    assert "coverage" in res["blocking_gates"]
+    cov = next(g for g in res["gates"] if g["gate"] == "coverage")
+    failed = [c for c in cov["checks"] if not c["passed"]]
+    assert failed and failed[0]["check"] == "coverage_rate"
+    assert cov["evidence"]["top_gaps"] is not None
+
+
+def test_scenario_quality_gate_blocks_on_impossible_threshold(mega_env,
+                                                              fresh_safety_root,
+                                                              eval_store):
+    overrides = {**LENIENT, "scenario_quality": {"min_geometric_pass_rate": 1.01}}
+    res = gates.evaluate_gates(mega_env["store"], mega_env["good"], mega_env["good"],
+                               policy_overrides=overrides)
+    assert "scenario_quality" in res["blocking_gates"]
+    sq = next(g for g in res["gates"] if g["gate"] == "scenario_quality")
+    assert sq["evidence"]["annotations_sampled"] > 0
+
+
+def test_evidence_package_structure_and_wording(mega_env, fresh_safety_root, eval_store):
+    gates.evaluate_gates(mega_env["store"], mega_env["good"], mega_env["good"],
+                         policy_overrides=LENIENT)
+    run_id = mega_env["good"].run_id
+    pkg = gates.load_evidence(run_id)
+    assert pkg is not None
+    assert pkg["package_id"] == f"sep-{run_id}"
+    # precise wording: supports a safety case, does not certify
+    assert "SUPPORT a safety case" in pkg["disclaimer"]
+    assert "not certify" in pkg["disclaimer"]
+    # standard mappings cover the three families
+    standards = " ".join(m["standard"] for m in pkg["standard_mappings"])
+    for token in ("ISO 26262", "21448", "UL 4600"):
+        assert token in standards
+    # methodology + lineage present
+    assert pkg["sampling_methodology"]["config"] is not None
+    assert pkg["lineage"]["candidate"]["evaluation_id"] == run_id
+    assert pkg["decision"]["release_ready"] is True
+
+    # persisted gate result retrievable
+    result = gates.latest_gate_result(run_id)
+    assert result is not None and result["evidence_package_id"] == pkg["package_id"]
+
+    # markdown rendering
+    md = gates.render_markdown(pkg)
+    assert "# Safety Evidence Package" in md
+    assert "RELEASE READY" in md
+    assert "Standard mappings" in md
+    assert pkg["disclaimer"][:40] in md

tests/test_safety/test_odd.py

@@ -0,0 +1,172 @@
+"""ODD taxonomy + combinatorial coverage math (exact expectations on a
+hand-built cube) + run-level coverage and gap filling."""
+
+from __future__ import annotations
+
+import math
+
+import pandas as pd
+import pytest
+
+from sensorflow.megaeval.cube import STAT_COLS
+from sensorflow.megaeval.population import DIM_NAMES, DIMENSIONS
+from sensorflow.safety import odd
+
+
+def test_taxonomy_structure():
+    tax = odd.taxonomy()
+    dims = tax["dimensions"]
+    for name in DIM_NAMES:
+        assert dims[name]["values"] == DIMENSIONS[name]
+        assert dims[name]["instrumented"] is True
+        assert dims[name]["iso34503_category"] in (
+            "scenery", "environmental_conditions", "dynamic_elements",
+            "sensor_configuration")
+    # declared-but-not-instrumented dimension is honest about itself
+    assert dims["geography"]["instrumented"] is False
+    assert "ISO 34503" in tax["standard_basis"]
+
+
+def _mini_cube() -> pd.DataFrame:
+    """Cube with data in exactly two weather cells:
+    clear: tp=90 fn=10 (n=100), rain: tp=8 fn=2 (n=10). fog/snow empty."""
+    rows = []
+    for weather_code, tp, fn in ((0, 90, 10), (1, 8, 2)):
+        row = {d: 0 for d in DIM_NAMES}
+        row["weather"] = weather_code
+        row.update({c: 0 for c in STAT_COLS})
+        row.update({"n": tp + fn, "tp": tp, "fn": fn, "sum_iou": 0.8 * tp})
+        rows.append(row)
+    return pd.DataFrame(rows)
+
+
+def _wilson(k, n, z=1.959963984540054):
+    if n == 0:
+        return 0.0, 1.0
+    p = k / n
+    denom = 1 + z ** 2 / n
+    center = (p + z ** 2 / (2 * n)) / denom
+    margin = z * math.sqrt(p * (1 - p) / n + z ** 2 / (4 * n ** 2)) / denom
+    return max(0.0, center - margin), min(1.0, center + margin)
+
+
+def test_coverage_exact_math_on_known_population():
+    dim_counts = {"weather": {"clear": 700, "rain": 200, "fog": 80, "snow": 20}}
+    cov = odd.compute_coverage(_mini_cube(), dims=["weather"], dim_counts=dim_counts,
+                               min_samples=20, max_ci_width=0.25, target_recall=0.9)
+    cells = {c["cell"]["weather"]: c for c in cov["cells"]}
+    assert set(cells) == {"clear", "rain", "fog", "snow"}
+
+    # clear: n=100, recall 0.9, Wilson CI hand-checked, adequate
+    clear = cells["clear"]
+    lo, hi = _wilson(90, 100)
+    assert clear["n"] == 100
+    assert clear["recall"] == pytest.approx(0.9)
+    assert clear["wilson_ci"][0] == pytest.approx(lo, abs=2e-3)
+    assert clear["wilson_ci"][1] == pytest.approx(hi, abs=2e-3)
+    assert clear["adequate"] and not clear["is_gap"]
+    assert clear["production_share"] == pytest.approx(0.7)
+    # risk = share * max(0, target - wilson_lo)
+    assert clear["risk"] == pytest.approx(0.7 * max(0.0, 0.9 - lo), abs=2e-3)
+
+    # rain: n=10 < 20 -> insufficient samples gap
+    rain = cells["rain"]
+    assert rain["is_gap"] and "insufficient_samples" in rain["gap_reasons"]
+    assert rain["recall"] == pytest.approx(0.8)
+
+    # empty cells: both gap reasons, CI width 1
+    fog = cells["fog"]
+    assert fog["n"] == 0 and fog["recall"] is None
+    assert set(fog["gap_reasons"]) == {"insufficient_samples", "ci_too_wide"}
+    assert fog["ci_width"] == pytest.approx(1.0)
+
+    # summary: exactly 1 adequate of 4 cells; weighted coverage = clear share
+    s = cov["summary"]
+    assert s["total_cells"] == 4
+    assert s["adequate_cells"] == 1 and s["gap_cells"] == 3
+    assert s["coverage_rate"] == pytest.approx(0.25)
+    assert s["production_weighted_coverage"] == pytest.approx(0.7)
+
+    # gaps ranked by risk = share * deficit: rain(share .2) > fog(.08) > snow(.02)
+    gap_order = [g["cell"]["weather"] for g in cov["gaps"]]
+    assert gap_order == ["rain", "fog", "snow"]
+    for g in cov["gaps"]:
+        assert g["risk"] == pytest.approx(
+            g["production_share"] * g["performance_deficit"], abs=1e-4)
+
+    # every gap gets a fulfillable request
+    assert len(cov["fill_requests"]) == 3
+    req = cov["fill_requests"][0]
+    assert req["cell"]["weather"] == "rain"
+    assert req["needed_samples"] == 10
+    assert req["generator"] == "sensorflow.evaluation.synthetic.generate_dataset"
+
+
+def test_coverage_rejects_oversized_cell_product():
+    with pytest.raises(ValueError):
+        odd.compute_coverage(_mini_cube(), dims=DIM_NAMES)  # way over 2000 cells
+
+
+def test_coverage_for_run_consistency(mega_env, safety_root):
+    cov = odd.coverage_for_run(mega_env["store"], mega_env["good"],
+                               dims=["weather", "lighting"], include_cells=True)
+    s = cov["summary"]
+    assert s["total_cells"] == len(DIMENSIONS["weather"]) * len(DIMENSIONS["lighting"])
+    assert s["populated_cells"] + s["empty_cells"] == s["total_cells"]
+    assert s["adequate_cells"] + s["gap_cells"] == s["total_cells"]
+    # cell mass equals the population's object count (every object in one cell)
+    assert sum(c["n"] for c in cov["cells"]) == mega_env["meta"]["num_objects"]
+    # adequacy consistent with the thresholds echoed in the response
+    th = cov["thresholds"]
+    for c in cov["cells"]:
+        expected = c["n"] >= th["min_samples"] and c["ci_width"] <= th["max_ci_width"]
+        assert c["adequate"] == expected
+    # gaps sorted by descending risk
+    risks = [g["risk"] for g in cov["gaps"]]
+    assert risks == sorted(risks, reverse=True)
+
+
+def test_fill_gap_generates_data_and_improves_cell(mega_env, fresh_safety_root, tmp_path):
+    from sensorflow.evaluation.records import reset_store
+    reset_store(tmp_path / "eval")
+
+    store, run = mega_env["store"], mega_env["good"]
+    cov = odd.coverage_for_run(store, run, dims=["weather", "lighting"],
+                               include_cells=True, min_samples=10_000)
+    gap = cov["gaps"][0]  # min_samples=10k guarantees gaps exist
+
+    result = odd.fill_gap(store, run, gap["cell"], num_sequences=1,
+                          frames_per_sequence=10)
+    assert result["objects_added"] > 0
+    assert result["cell_after"]["n"] > result["cell_before"]["n"]
+    assert result["cell_after"]["n"] == (result["cell_before"]["n"]
+                                         + result["objects_added"])
+    assert result["supplement"]["source"] == "synthetic"
+
+    # supplement persisted and folded into subsequent coverage computations
+    supplements = odd.load_supplements(run.run_id)
+    assert len(supplements) == 1
+    cov2 = odd.coverage_for_run(store, run, dims=["weather", "lighting"],
+                                include_cells=True, min_samples=10_000)
+    cell2 = next(c for c in cov2["cells"] if c["cell_id"] == gap["cell_id"])
+    assert cell2["n"] == result["cell_after"]["n"]
+    assert cell2["synthetic_supplement"]["n_added"] == result["objects_added"]
+
+    # generated frames carry the requested conditions
+    from sensorflow.evaluation.records import get_store
+    legacy = get_store()
+    frames = legacy.where("frames", dataset_id=result["generated_dataset_id"])
+    assert frames
+    weather = gap["cell"].get("weather")
+    if weather:
+        assert all(f.weather == weather for f in frames)
+
+    # scenario DB registered the synthetic gap-fill scenario
+    from sensorflow.safety.scenario_db import get_db
+    recs = get_db().search(source="synthetic")
+    assert any(r.scenario_type == "odd_gap_fill" for r in recs)
+
+
+def test_fill_gap_rejects_unknown_cell(mega_env, fresh_safety_root):
+    with pytest.raises(ValueError):
+        odd.fill_gap(mega_env["store"], mega_env["good"], {"weather": "hurricane"})

tests/test_safety/test_scenario_db.py

@@ -0,0 +1,87 @@
+"""Scenario database: populate (idempotently) from rare events, search/filter,
+export bundle."""
+
+from __future__ import annotations
+
+from sensorflow.evaluation.records import RareEvent
+from sensorflow.safety.scenario_db import ScenarioRecord, get_db
+
+
+def _make_rare_event(store, ds, frame, severity="high"):
+    ev = RareEvent(
+        event_id=f"evt-test-{severity}",
+        dataset_id=ds.dataset_id,
+        scenario_type="near_collision",
+        severity=severity,
+        rarity_score=0.9,
+        confidence=0.8,
+        evidence_frames=[frame.frame_id],
+        description="test rare event",
+    )
+    store.put("rare_events", ev)
+    return ev
+
+
+def test_populate_from_rare_events_idempotent(eval_env, fresh_safety_root):
+    store, ds = eval_env
+    frame = store.where("frames", dataset_id=ds.dataset_id)[0]
+    _make_rare_event(store, ds, frame)
+
+    db = get_db()
+    assert db.add_from_rare_events(store) == 1
+    assert db.add_from_rare_events(store) == 0  # idempotent
+
+    recs = db.search(source="rare_event")
+    assert len(recs) == 1
+    r = recs[0]
+    assert r.scenario_type == "near_collision"
+    assert r.odd_tags.get("weather") == frame.weather
+    assert f"rare_event:{'evt-test-high'}" in r.evidence_refs
+
+
+def test_search_filters_and_ordering(fresh_safety_root):
+    db = get_db()
+    db.add(ScenarioRecord(scenario_id="s1", scenario_type="cut_in", source="mined",
+                          severity="low", odd_tags={"weather": "rain"},
+                          description="cut-in in rain"))
+    db.add(ScenarioRecord(scenario_id="s2", scenario_type="cut_in", source="synthetic",
+                          severity="critical", odd_tags={"weather": "clear"},
+                          description="synthetic cut-in"))
+    db.add(ScenarioRecord(scenario_id="s3", scenario_type="jaywalk", source="mined",
+                          severity="high", odd_tags={"weather": "rain"},
+                          description="pedestrian jaywalking at night"))
+
+    assert {r.scenario_id for r in db.search(scenario_type="cut_in")} == {"s1", "s2"}
+    assert {r.scenario_id for r in db.search(source="mined")} == {"s1", "s3"}
+    assert {r.scenario_id for r in db.search(severity="critical")} == {"s2"}
+    assert {r.scenario_id for r in db.search(odd_tags={"weather": "rain"})} == {"s1", "s3"}
+    assert {r.scenario_id for r in db.search(text="jaywalk")} == {"s3"}
+    # ordering: highest severity first
+    assert [r.scenario_id for r in db.search()][:2] == ["s2", "s3"]
+
+    counts = db.counts()
+    assert counts["total"] == 3
+    assert counts["by_source"] == {"mined": 2, "synthetic": 1}
+
+
+def test_persistence_across_reload(fresh_safety_root):
+    db = get_db()
+    db.add(ScenarioRecord(scenario_id="sp1", scenario_type="cut_in", source="mined",
+                          severity="medium"))
+    from sensorflow.safety import scenario_db as mod
+    mod.reset_db()  # force reload from disk
+    assert [r.scenario_id for r in get_db().search()] == ["sp1"]
+
+
+def test_export_bundle(fresh_safety_root):
+    db = get_db()
+    db.add(ScenarioRecord(scenario_id="e1", scenario_type="cut_in", source="mined",
+                          severity="high"))
+    db.add(ScenarioRecord(scenario_id="e2", scenario_type="jaywalk",
+                          source="discrepancy", severity="low"))
+    bundle = db.export_bundle(source="mined")
+    assert bundle["bundle_format"] == "sensorflow-scenario-bundle/v1"
+    assert bundle["count"] == 1
+    assert bundle["scenarios"][0]["scenario_id"] == "e1"
+    full = db.export_bundle()
+    assert full["count"] == 2

tests/test_safety/test_semantic.py

@@ -0,0 +1,71 @@
+"""Neuro-symbolic semantic mining: deterministic offline ranking with per-stage
+explanations, symbolic filters, and hybrid retrieval over containers +
+scenario-DB records."""
+
+from __future__ import annotations
+
+from sensorflow.safety import semantic
+from sensorflow.safety.scenario_db import ScenarioRecord, get_db
+
+
+def test_container_search_deterministic_and_explained(mega_env, safety_root):
+    store, run = mega_env["store"], mega_env["good"]
+    q = "pedestrian near miss at night in rain"
+    r1 = semantic.search_containers(store, run, q, k=10, use_llm=False)
+    r2 = semantic.search_containers(store, run, q, k=10, use_llm=False)
+    assert r1["results"] == r2["results"]  # deterministic offline ranking
+
+    assert r1["stage2"]["provider"] == "offline_deterministic"
+    results = r1["results"]
+    assert results
+    scores = [r["score"] for r in results]
+    assert scores == sorted(scores, reverse=True)
+    top = results[0]
+    # concept terms must be reflected in structured evidence
+    assert top["lighting"] == "night"
+    assert top["weather"] != "clear"
+    reasons = top["explanations"]["stage2_reasoning"]
+    assert reasons and all({"term", "meaning", "evidence"} <= set(m) for m in reasons)
+    # hybrid retrieval blended embedding similarity
+    assert r1["hybrid"]["embedding_weight"] > 0
+    assert top["embedding_similarity"] is not None
+
+
+def test_container_search_symbolic_filter(mega_env, safety_root):
+    store, run = mega_env["store"], mega_env["good"]
+    res = semantic.search_containers(store, run, "risky scenes",
+                                     filters={"lighting": "night"}, k=20,
+                                     use_llm=False)
+    assert res["stage1"]["applied_filters"] == {"lighting": ["night"]}
+    assert res["stage1"]["candidates_after"] < res["stage1"]["candidates_before"]
+    assert all(r["lighting"] == "night" for r in res["results"])
+
+
+def test_container_search_fallback_on_unmatched_concept(mega_env, safety_root):
+    store, run = mega_env["store"], mega_env["good"]
+    res = semantic.search_containers(store, run, "xylophone zeitgeist", k=5,
+                                     use_llm=False)
+    assert res["results"]
+    reasons = res["results"][0]["explanations"]["stage2_reasoning"]
+    assert any("no lexicon term matched" in m["meaning"] for m in reasons)
+
+
+def test_scenario_search_ranks_matching_odd_tags(fresh_safety_root):
+    db = get_db()
+    db.add(ScenarioRecord(scenario_id="n1", scenario_type="near_miss",
+                          source="mined", severity="high",
+                          odd_tags={"weather": "rain", "lighting": "night",
+                                    "actor_class": "pedestrian"},
+                          description="pedestrian near miss in night rain"))
+    db.add(ScenarioRecord(scenario_id="d1", scenario_type="nominal",
+                          source="synthetic", severity="low",
+                          odd_tags={"weather": "clear", "lighting": "day"},
+                          description="clear day nominal drive"))
+
+    res = semantic.search_scenarios("pedestrian near miss at night in rain", k=5)
+    assert res["results"][0]["scenario_id"] == "n1"
+    assert res["results"][0]["score"] > res["results"][-1]["score"]
+    assert res["stage2"]["provider"] == "offline_deterministic"
+
+    filtered = semantic.search_scenarios("anything", filters={"source": "synthetic"})
+    assert [r["scenario_id"] for r in filtered["results"]] == ["d1"]

tests/test_safety/test_ssam.py

@@ -0,0 +1,144 @@
+"""Extended SSAM measures against hand-computed values (TTC, DRAC, collision
+probability, DeltaS/MaxS, CSI aggregation, zone-grid PET) plus the synthetic
+scenario suite and model-conditioned CSI."""
+
+from __future__ import annotations
+
+import math
+
+import pytest
+
+from sensorflow.safety import ssam_ext
+
+CAR = (4.5, 1.9)
+
+
+def _state(t, x, y, speed, heading=0.0):
+    return {"t": t, "x": x, "y": y, "speed": speed, "heading": heading}
+
+
+def test_projected_ttc_hand_computed():
+    # A at x=0 doing 10 m/s toward stationary B at x=20, both 4.5 m long:
+    # bumper gap = 20 - 2.25 - 2.25 = 15.5 m -> TTC = 1.55 s.
+    a = _state(0.0, 0.0, 0.0, 10.0)
+    b = _state(0.0, 20.0, 0.0, 0.0)
+    ttc = ssam_ext.projected_ttc(a, CAR, b, CAR, look_ahead=3.0, dt=0.05)
+    assert ttc == pytest.approx(1.55, abs=0.02)
+
+    # diverging pair -> no TTC
+    b_away = _state(0.0, 20.0, 0.0, 15.0)  # same heading, faster, pulling away
+    assert ssam_ext.projected_ttc(a, CAR, b_away, CAR, 3.0, 0.05) is None
+
+
+def test_drac_hand_computed():
+    # closing speed 10 m/s, gap 15.5 m -> DRAC = 10^2 / (2*15.5) = 3.2258 m/s^2
+    a = _state(0.0, 0.0, 0.0, 10.0)
+    b = _state(0.0, 20.0, 0.0, 0.0)
+    assert ssam_ext.drac(a, CAR, b, CAR) == pytest.approx(100 / 31, abs=1e-6)
+    # opening geometry -> None
+    a_rev = _state(0.0, 0.0, 0.0, 10.0, heading=math.pi)
+    assert ssam_ext.drac(a_rev, CAR, b, CAR) is None
+
+
+def test_collision_probability_formula():
+    assert ssam_ext.collision_probability(None, 1.5) == 0.0
+    assert ssam_ext.collision_probability(1.5, 1.5) == 0.0
+    assert ssam_ext.collision_probability(0.75, 1.5) == pytest.approx(0.5)
+    assert ssam_ext.collision_probability(0.0, 1.5) == 1.0
+    assert ssam_ext.collision_probability(9.0, 1.5) == 0.0  # clipped
+
+
+def _head_on_pair(dt=0.1, duration=2.0):
+    """A drives at 10 m/s toward stationary B at x=30 (same lane)."""
+    a = {"vehicle_id": "A", "vehicle_type": "car", "length": 4.5, "width": 1.9,
+         "states": [_state(round(i * dt, 6), 10.0 * i * dt, 0.0, 10.0)
+                    for i in range(int(duration / dt) + 1)]}
+    b = {"vehicle_id": "B", "vehicle_type": "car", "length": 4.5, "width": 1.9,
+         "states": [_state(round(i * dt, 6), 30.0, 0.0, 0.0)
+                    for i in range(int(duration / dt) + 1)]}
+    return a, b
+
+
+def test_analyze_trajectories_measures_and_csi_aggregation():
+    a, b = _head_on_pair()
+    res = ssam_ext.analyze_trajectories([a, b])
+    agg = res["aggregate"]
+    assert agg["num_conflicts"] == 1
+    conflict = res["conflicts"][0]
+
+    # severity proxies: relative speed 10, max abs speed 10, rear-end geometry
+    assert conflict["delta_s_mps"] == pytest.approx(10.0, abs=1e-6)
+    assert conflict["max_s_mps"] == pytest.approx(10.0, abs=1e-6)
+    assert conflict["conflict_type"] == "rear_end"
+    assert 0.0 < conflict["collision_probability"] <= 1.0
+
+    # CSI aggregation re-derived with an independent loop over the same
+    # primitives: CSI = sum 0.5 * mu * DeltaS^2 * p(t) * dt   (mu reduced mass)
+    p = res["params"]
+    mu = 0.5  # two cars: 1*1/(1+1)
+    expected = 0.0
+    for sa, sb in zip(a["states"], b["states"]):
+        ttc = ssam_ext.projected_ttc(sa, CAR, sb, CAR,
+                                     p["look_ahead_s"], p["projection_dt_s"])
+        if ttc is None or ttc > p["ttc_threshold_s"]:
+            continue
+        prob = ssam_ext.collision_probability(ttc, p["ttc_threshold_s"])
+        expected += 0.5 * mu * 10.0 ** 2 * prob * 0.1
+    assert agg["aggregate_csi"] == pytest.approx(expected, rel=1e-6)
+    assert agg["aggregate_csi"] > 0
+
+
+def test_zone_grid_pet_hand_computed():
+    # A crosses the origin cell around t=1.0 (moving +x through x=0),
+    # B crosses the same spot around t=3.0 (moving +y through y=0):
+    # PET ~= 2.0 s (grid quantization tolerance from finite footprints).
+    dt = 0.1
+    a = {"vehicle_id": "A", "length": 1.0, "width": 1.0,
+         "states": [_state(round(i * dt, 6), -10.0 + 10.0 * i * dt, 0.0, 10.0)
+                    for i in range(21)]}
+    b = {"vehicle_id": "B", "length": 1.0, "width": 1.0,
+         "states": [{"t": round(2.0 + i * dt, 6), "x": 0.0, "y": -10.0 + 10.0 * i * dt,
+                     "speed": 10.0, "heading": math.pi / 2} for i in range(21)]}
+    pet = ssam_ext.zone_grid_pet(a, b, cell=0.5)
+    assert pet is not None
+    assert pet == pytest.approx(2.0, abs=0.35)
+
+
+def test_scenario_suite_produces_conflicts():
+    for scenario in ("crossing", "rear_end", "lane_change"):
+        trajs = ssam_ext.generate_trajectories(seed=0, scenario=scenario,
+                                               reaction_delay_s=0.6)
+        res = ssam_ext.analyze_trajectories(trajs)
+        assert res["aggregate"]["num_conflicts"] >= 1, scenario
+        assert res["aggregate"]["aggregate_csi"] > 0, scenario
+    with pytest.raises(ValueError):
+        ssam_ext.generate_trajectories(scenario="teleportation")
+
+
+def test_reaction_delay_monotonic_and_csi_increases():
+    base = ssam_ext.reaction_delay_for_model("model-v41")
+    worse = ssam_ext.reaction_delay_for_model(
+        "model-v42-regressed", overrides={"night_penalty": 0.35, "vru_penalty": 0.12})
+    assert worse["reaction_delay_s"] > base["reaction_delay_s"]
+    assert base["simulated"] is True
+
+    csis = []
+    for delay in (base["reaction_delay_s"], worse["reaction_delay_s"]):
+        total = 0.0
+        for scenario in ("crossing", "rear_end", "lane_change"):
+            trajs = ssam_ext.generate_trajectories(seed=0, scenario=scenario,
+                                                   reaction_delay_s=delay)
+            total += ssam_ext.analyze_trajectories(trajs)["aggregate"]["aggregate_csi"]
+        csis.append(total)
+    assert csis[1] > csis[0]
+
+
+def test_csi_for_run_cached_and_ordered(mega_env, safety_root):
+    good = ssam_ext.csi_for_run(mega_env["good"])
+    bad = ssam_ext.csi_for_run(mega_env["bad"])
+    assert good["aggregate_csi"] > 0
+    assert bad["aggregate_csi"] > good["aggregate_csi"]
+    assert good["simulated"] is True
+    # cached second read is identical
+    again = ssam_ext.csi_for_run(mega_env["good"])
+    assert again == good

tests/test_seqeval/conftest.py

@@ -0,0 +1,36 @@
+"""Shared fixtures: isolated megaeval + seqeval roots and one mid-size
+population reused by every seqeval test module."""
+
+from __future__ import annotations
+
+import pytest
+
+from sensorflow.megaeval import population as pop_mod
+from sensorflow.seqeval import ledger as ledger_mod
+from sensorflow.seqeval import paired as paired_mod
+from sensorflow.seqeval.controller import reset_seqeval_store
+
+
+@pytest.fixture(scope="session")
+def seq_env(tmp_path_factory):
+    root = tmp_path_factory.mktemp("seqeval_env")
+    pop_mod.set_mega_root(str(root / "mega"))
+    ledger_mod.set_seqeval_root(str(root / "seq"))
+    reset_seqeval_store()
+    paired_mod.reset_prediction_cache()
+    meta = pop_mod.generate_population("seq-pop", num_objects=90_000, seed=11)
+    yield {"root": root, "meta": meta}
+    pop_mod.set_mega_root("runs/megaeval")
+    ledger_mod.set_seqeval_root("runs/seqeval")
+    reset_seqeval_store()
+    paired_mod.reset_prediction_cache()
+
+
+# A compact policy for fast Monte-Carlo style tests.
+FAST_POLICY = {
+    "target_n": 6000,
+    "safety_floor": 800,
+    "min_per_stratum": 120,
+    "escalation": {"enabled": True, "max_extra_per_stratum": 600,
+                   "batch_objects": 300},
+}