Spaces:
Paused
Paused
Deploy Sensorflow Studio
Browse files- .gitignore +1 -0
- app_backend.py +4 -0
- sensorflow/platform/__init__.py +15 -0
- sensorflow/platform/api.py +237 -0
- sensorflow/platform/compare.py +122 -0
- sensorflow/platform/container_quality.py +132 -0
- sensorflow/platform/entities.py +306 -0
- sensorflow/platform/evidence.py +140 -0
- sensorflow/platform/gates.py +264 -0
- sensorflow/platform/levels.py +112 -0
- sensorflow/platform/metrics_engine.py +163 -0
- src/components/platform/ContainerQualityPanel.tsx +94 -0
- src/components/platform/GateStatusPanel.tsx +84 -0
- src/components/platform/ModelComparePanel.tsx +117 -0
- src/pages/CommandCenterPage.tsx +33 -8
- src/pages/EvaluationPage.tsx +2 -0
- src/pages/QualityEnginePage.tsx +2 -0
- src/services/platform.ts +165 -0
- tests/test_platform/__init__.py +1 -0
- tests/test_platform/test_foundation.py +285 -0
.gitignore
CHANGED
|
@@ -65,6 +65,7 @@ temp/
|
|
| 65 |
# Generated runtime/test artifacts
|
| 66 |
runs/labeleval/
|
| 67 |
runs/megaeval/
|
|
|
|
| 68 |
runs/pipeline/frame_test/
|
| 69 |
runs/pipeline/qg_test/
|
| 70 |
runs/pipeline/roundtrip/
|
|
|
|
| 65 |
# Generated runtime/test artifacts
|
| 66 |
runs/labeleval/
|
| 67 |
runs/megaeval/
|
| 68 |
+
runs/platform/
|
| 69 |
runs/pipeline/frame_test/
|
| 70 |
runs/pipeline/qg_test/
|
| 71 |
runs/pipeline/roundtrip/
|
app_backend.py
CHANGED
|
@@ -1467,6 +1467,10 @@ app.include_router(labeleval_router)
|
|
| 1467 |
from sensorflow.megaeval.api import router as megaeval_router
|
| 1468 |
app.include_router(megaeval_router)
|
| 1469 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1470 |
# Mount static folder
|
| 1471 |
static_dir = Path(__file__).parent / "static"
|
| 1472 |
if static_dir.exists():
|
|
|
|
| 1467 |
from sensorflow.megaeval.api import router as megaeval_router
|
| 1468 |
app.include_router(megaeval_router)
|
| 1469 |
|
| 1470 |
+
# Population-scale foundation (container quality, multi-model compare, multi-gate, evidence).
|
| 1471 |
+
from sensorflow.platform.api import router as platform_router
|
| 1472 |
+
app.include_router(platform_router)
|
| 1473 |
+
|
| 1474 |
# Mount static folder
|
| 1475 |
static_dir = Path(__file__).parent / "static"
|
| 1476 |
if static_dir.exists():
|
sensorflow/platform/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Population-scale evaluation foundation — extends MegaEval + LabelEval.
|
| 2 |
+
|
| 3 |
+
Phase 1: schemas, aggregate levels, metrics, container quality, model compare,
|
| 4 |
+
multi-gate skeleton, evidence packages. Phases 2–7 leave TODO hooks only.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from sensorflow.platform.levels import AggregateLevel, EvaluationScope
|
| 8 |
+
from sensorflow.platform.entities import Provenance, EvidencePackage
|
| 9 |
+
|
| 10 |
+
__all__ = [
|
| 11 |
+
"AggregateLevel",
|
| 12 |
+
"EvaluationScope",
|
| 13 |
+
"Provenance",
|
| 14 |
+
"EvidencePackage",
|
| 15 |
+
]
|
sensorflow/platform/api.py
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Platform Phase 1 FastAPI routes — mount into existing app_backend.
|
| 2 |
+
|
| 3 |
+
Routes:
|
| 4 |
+
/api/evaluations/...
|
| 5 |
+
/api/containers/...
|
| 6 |
+
/api/models/compare
|
| 7 |
+
/api/gates/...
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
from typing import Any, Dict, List, Optional
|
| 13 |
+
|
| 14 |
+
from fastapi import APIRouter, HTTPException
|
| 15 |
+
from pydantic import BaseModel, Field
|
| 16 |
+
|
| 17 |
+
from sensorflow.platform import levels
|
| 18 |
+
from sensorflow.platform.container_quality import build_container_quality_profile
|
| 19 |
+
from sensorflow.platform.compare import compare_models
|
| 20 |
+
from sensorflow.platform.evidence import build_evidence_package, export_evidence_package
|
| 21 |
+
from sensorflow.platform.gates import evaluate_multi_gates, list_gate_defs, load_gate_config, save_gate_config
|
| 22 |
+
from sensorflow.platform.levels import AggregateLevel, EvaluationScope
|
| 23 |
+
|
| 24 |
+
router = APIRouter(prefix="/api")
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
# ------------------------------------------------------------------ requests
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class EvaluationCreateRequest(BaseModel):
|
| 31 |
+
level: AggregateLevel = AggregateLevel.POPULATION
|
| 32 |
+
dataset_id: Optional[str] = None
|
| 33 |
+
population_id: Optional[str] = None
|
| 34 |
+
run_id: Optional[str] = None
|
| 35 |
+
container_id: Optional[str] = None
|
| 36 |
+
mega_container_id: Optional[int] = None
|
| 37 |
+
drive_id: Optional[str] = None
|
| 38 |
+
scene_id: Optional[str] = None
|
| 39 |
+
sequence_id: Optional[str] = None
|
| 40 |
+
frame_id: Optional[str] = None
|
| 41 |
+
cohort_filters: Dict[str, List[str]] = Field(default_factory=dict)
|
| 42 |
+
model_version: Optional[str] = None
|
| 43 |
+
label_version: Optional[str] = None
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class ModelCompareRequest(BaseModel):
|
| 47 |
+
run_ids: List[str] = Field(..., min_length=2)
|
| 48 |
+
baseline_run_id: Optional[str] = None
|
| 49 |
+
policy: Dict[str, Any] = Field(default_factory=dict)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class GateEvaluateRequest(BaseModel):
|
| 53 |
+
sequence_id: Optional[str] = None
|
| 54 |
+
candidate_run_id: Optional[str] = None
|
| 55 |
+
baseline_run_id: Optional[str] = None
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class GateConfigUpdate(BaseModel):
|
| 59 |
+
gates: Dict[str, Any] = Field(default_factory=dict)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class EvidenceExportRequest(BaseModel):
|
| 63 |
+
evaluation_id: Optional[str] = None
|
| 64 |
+
run_id: Optional[str] = None
|
| 65 |
+
dataset_id: Optional[str] = None
|
| 66 |
+
sequence_id: Optional[str] = None
|
| 67 |
+
candidate_run_id: Optional[str] = None
|
| 68 |
+
baseline_run_id: Optional[str] = None
|
| 69 |
+
persist: bool = True
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
# ------------------------------------------------------------------ evaluations
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
@router.get("/evaluations/levels")
|
| 76 |
+
def evaluation_levels():
|
| 77 |
+
return {
|
| 78 |
+
"levels": [lvl.value for lvl in AggregateLevel.ladder()],
|
| 79 |
+
"backends": {lvl.value: levels.LEVEL_BACKEND[lvl] for lvl in AggregateLevel},
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
@router.post("/evaluations/scope")
|
| 84 |
+
def create_evaluation_scope(req: EvaluationCreateRequest):
|
| 85 |
+
scope = EvaluationScope(**req.model_dump())
|
| 86 |
+
return {
|
| 87 |
+
"scope": levels.summarize_scope(scope),
|
| 88 |
+
"message": "Phase 1 scope descriptor — queries still served by MegaEval/LabelEval backends",
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
@router.get("/evaluations/{run_id}")
|
| 93 |
+
def get_evaluation_summary(run_id: str):
|
| 94 |
+
"""Unified evaluation summary for a MegaEval run (async job status preserved)."""
|
| 95 |
+
try:
|
| 96 |
+
from sensorflow.megaeval.runs import get_mega_store
|
| 97 |
+
run = get_mega_store().runs.get(run_id)
|
| 98 |
+
except Exception as exc:
|
| 99 |
+
raise HTTPException(500, str(exc)) from exc
|
| 100 |
+
if run is None:
|
| 101 |
+
raise HTTPException(404, f"Unknown evaluation run {run_id}")
|
| 102 |
+
scope = EvaluationScope(
|
| 103 |
+
level=AggregateLevel.POPULATION,
|
| 104 |
+
evaluation_id=run.run_id,
|
| 105 |
+
run_id=run.run_id,
|
| 106 |
+
population_id=run.population_id,
|
| 107 |
+
model_version=run.model_version,
|
| 108 |
+
label_version=getattr(run, "label_version", None),
|
| 109 |
+
)
|
| 110 |
+
return {
|
| 111 |
+
"evaluation_id": run.run_id,
|
| 112 |
+
"status": run.status,
|
| 113 |
+
"scope": levels.summarize_scope(scope),
|
| 114 |
+
"headline": run.headline,
|
| 115 |
+
"model_version": run.model_version,
|
| 116 |
+
"population_id": run.population_id,
|
| 117 |
+
"lineage": getattr(run, "lineage", None),
|
| 118 |
+
"progress": getattr(run, "progress", None),
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
@router.post("/evaluations/evidence")
|
| 123 |
+
def export_evidence(req: EvidenceExportRequest):
|
| 124 |
+
package = build_evidence_package(
|
| 125 |
+
evaluation_id=req.evaluation_id,
|
| 126 |
+
run_id=req.run_id,
|
| 127 |
+
dataset_id=req.dataset_id,
|
| 128 |
+
sequence_id=req.sequence_id,
|
| 129 |
+
candidate_run_id=req.candidate_run_id,
|
| 130 |
+
baseline_run_id=req.baseline_run_id,
|
| 131 |
+
)
|
| 132 |
+
path = None
|
| 133 |
+
if req.persist:
|
| 134 |
+
path = str(export_evidence_package(package))
|
| 135 |
+
return {"package": package.model_dump(), "path": path}
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
# ------------------------------------------------------------------ containers
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
@router.get("/containers/quality")
|
| 142 |
+
def containers_quality(
|
| 143 |
+
run_id: str,
|
| 144 |
+
sort: str = "highest_risk",
|
| 145 |
+
limit: int = 50,
|
| 146 |
+
offset: int = 0,
|
| 147 |
+
):
|
| 148 |
+
try:
|
| 149 |
+
return build_container_quality_profile(
|
| 150 |
+
run_id, sort=sort, limit=limit, offset=offset
|
| 151 |
+
)
|
| 152 |
+
except KeyError as exc:
|
| 153 |
+
raise HTTPException(404, str(exc)) from exc
|
| 154 |
+
except RuntimeError as exc:
|
| 155 |
+
raise HTTPException(409, str(exc)) from exc
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
@router.get("/containers/{container_id}/quality")
|
| 159 |
+
def container_quality(container_id: int, run_id: str):
|
| 160 |
+
try:
|
| 161 |
+
return build_container_quality_profile(run_id, container_id=container_id)
|
| 162 |
+
except KeyError as exc:
|
| 163 |
+
raise HTTPException(404, str(exc)) from exc
|
| 164 |
+
except RuntimeError as exc:
|
| 165 |
+
raise HTTPException(409, str(exc)) from exc
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
# ------------------------------------------------------------------ models compare
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
@router.post("/models/compare")
|
| 172 |
+
def models_compare(req: ModelCompareRequest):
|
| 173 |
+
try:
|
| 174 |
+
return compare_models(
|
| 175 |
+
req.run_ids,
|
| 176 |
+
baseline_run_id=req.baseline_run_id,
|
| 177 |
+
policy=req.policy or None,
|
| 178 |
+
)
|
| 179 |
+
except ValueError as exc:
|
| 180 |
+
raise HTTPException(400, str(exc)) from exc
|
| 181 |
+
except KeyError as exc:
|
| 182 |
+
raise HTTPException(404, str(exc)) from exc
|
| 183 |
+
except RuntimeError as exc:
|
| 184 |
+
raise HTTPException(409, str(exc)) from exc
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
# ------------------------------------------------------------------ gates
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
@router.get("/gates")
|
| 191 |
+
def gates_list():
|
| 192 |
+
cfg = load_gate_config()
|
| 193 |
+
return {
|
| 194 |
+
"config_path": str(cfg.get("schema_version")),
|
| 195 |
+
"gates": [g.model_dump() for g in list_gate_defs(cfg)],
|
| 196 |
+
"order": ["scenario", "coverage", "regression", "safety", "quality", "launch", "release"],
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
@router.get("/gates/config")
|
| 201 |
+
def gates_config():
|
| 202 |
+
return load_gate_config()
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
@router.post("/gates/config")
|
| 206 |
+
def gates_config_update(req: GateConfigUpdate):
|
| 207 |
+
cfg = load_gate_config()
|
| 208 |
+
for name, patch in req.gates.items():
|
| 209 |
+
if name not in cfg["gates"]:
|
| 210 |
+
cfg["gates"][name] = patch
|
| 211 |
+
else:
|
| 212 |
+
cfg["gates"][name].update(patch)
|
| 213 |
+
save_gate_config(cfg)
|
| 214 |
+
return cfg
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
@router.post("/gates/evaluate")
|
| 218 |
+
def gates_evaluate(req: GateEvaluateRequest):
|
| 219 |
+
return evaluate_multi_gates(
|
| 220 |
+
sequence_id=req.sequence_id,
|
| 221 |
+
candidate_run_id=req.candidate_run_id,
|
| 222 |
+
baseline_run_id=req.baseline_run_id,
|
| 223 |
+
)
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
@router.get("/gates/status")
|
| 227 |
+
def gates_status(
|
| 228 |
+
sequence_id: Optional[str] = None,
|
| 229 |
+
candidate_run_id: Optional[str] = None,
|
| 230 |
+
baseline_run_id: Optional[str] = None,
|
| 231 |
+
):
|
| 232 |
+
"""Multi-gate readiness skeleton for UI cards."""
|
| 233 |
+
return evaluate_multi_gates(
|
| 234 |
+
sequence_id=sequence_id,
|
| 235 |
+
candidate_run_id=candidate_run_id,
|
| 236 |
+
baseline_run_id=baseline_run_id,
|
| 237 |
+
)
|
sensorflow/platform/compare.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Multi-model comparison (A vs B vs C) wrapping MegaEval comparative analytics.
|
| 2 |
+
|
| 3 |
+
Does not invent a second compare engine — delegates pairwise compare_runs and
|
| 4 |
+
folds results into a multi-way delta table.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import Any, Dict, List, Optional
|
| 10 |
+
|
| 11 |
+
from sensorflow.megaeval import analysis
|
| 12 |
+
from sensorflow.megaeval.runs import MegaStore, get_mega_store
|
| 13 |
+
from sensorflow.platform.metrics_engine import delta_metrics, round4
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _headline_map(run) -> Dict[str, Optional[float]]:
|
| 17 |
+
h = run.headline or {}
|
| 18 |
+
return {k: h.get(k) for k in (
|
| 19 |
+
"precision", "recall", "f1", "mean_iou", "safety_recall", "anomaly_rate"
|
| 20 |
+
)}
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def compare_models(
|
| 24 |
+
run_ids: List[str],
|
| 25 |
+
*,
|
| 26 |
+
baseline_run_id: Optional[str] = None,
|
| 27 |
+
policy: Optional[Dict] = None,
|
| 28 |
+
store: Optional[MegaStore] = None,
|
| 29 |
+
) -> Dict[str, Any]:
|
| 30 |
+
"""Compare 2+ published MegaEval runs (model versions A/B/C…).
|
| 31 |
+
|
| 32 |
+
If baseline_run_id is omitted, the first id is the baseline.
|
| 33 |
+
"""
|
| 34 |
+
if len(run_ids) < 2:
|
| 35 |
+
raise ValueError("compare_models requires at least two run_ids")
|
| 36 |
+
store = store or get_mega_store()
|
| 37 |
+
baseline_id = baseline_run_id or run_ids[0]
|
| 38 |
+
if baseline_id not in run_ids:
|
| 39 |
+
run_ids = [baseline_id, *run_ids]
|
| 40 |
+
|
| 41 |
+
runs = []
|
| 42 |
+
for rid in run_ids:
|
| 43 |
+
run = store.runs.get(rid)
|
| 44 |
+
if run is None:
|
| 45 |
+
raise KeyError(f"Unknown evaluation run {rid}")
|
| 46 |
+
if run.status != "published":
|
| 47 |
+
raise RuntimeError(f"Run {rid} is {run.status}; results require published")
|
| 48 |
+
runs.append(run)
|
| 49 |
+
|
| 50 |
+
baseline = next(r for r in runs if r.run_id == baseline_id)
|
| 51 |
+
candidates = [r for r in runs if r.run_id != baseline_id]
|
| 52 |
+
|
| 53 |
+
models = [{
|
| 54 |
+
"run_id": r.run_id,
|
| 55 |
+
"model_version": r.model_version,
|
| 56 |
+
"role": "baseline" if r.run_id == baseline_id else "candidate",
|
| 57 |
+
"headline": _headline_map(r),
|
| 58 |
+
} for r in runs]
|
| 59 |
+
|
| 60 |
+
# Multi-way metric table: one row per metric, columns per model
|
| 61 |
+
metric_keys = ["precision", "recall", "f1", "mean_iou", "safety_recall", "anomaly_rate"]
|
| 62 |
+
matrix = []
|
| 63 |
+
for key in metric_keys:
|
| 64 |
+
row: Dict[str, Any] = {"metric": key}
|
| 65 |
+
for r in runs:
|
| 66 |
+
row[r.model_version] = round4(r.headline.get(key)) if r.headline.get(key) is not None else None
|
| 67 |
+
base_v = baseline.headline.get(key)
|
| 68 |
+
deltas = {}
|
| 69 |
+
for c in candidates:
|
| 70 |
+
cv = c.headline.get(key)
|
| 71 |
+
if base_v is not None and cv is not None:
|
| 72 |
+
deltas[c.model_version] = round4(float(cv) - float(base_v))
|
| 73 |
+
row["deltas_vs_baseline"] = deltas
|
| 74 |
+
matrix.append(row)
|
| 75 |
+
|
| 76 |
+
pairwise = []
|
| 77 |
+
recommendations = []
|
| 78 |
+
for cand in candidates:
|
| 79 |
+
cmp = analysis.compare_runs(store, cand, baseline, policy)
|
| 80 |
+
pairwise.append({
|
| 81 |
+
"baseline_run_id": baseline.run_id,
|
| 82 |
+
"candidate_run_id": cand.run_id,
|
| 83 |
+
"baseline_model": baseline.model_version,
|
| 84 |
+
"candidate_model": cand.model_version,
|
| 85 |
+
"recommendation": cmp.get("recommendation"),
|
| 86 |
+
"blockers": cmp.get("blockers") or [],
|
| 87 |
+
"headline": cmp.get("headline_deltas") or delta_metrics(
|
| 88 |
+
_headline_map(baseline), _headline_map(cand)
|
| 89 |
+
),
|
| 90 |
+
"per_class": cmp.get("per_class") or [],
|
| 91 |
+
"regressions": (cmp.get("regressions") or cmp.get("worst_cohorts") or [])[:10],
|
| 92 |
+
})
|
| 93 |
+
recommendations.append({
|
| 94 |
+
"candidate": cand.model_version,
|
| 95 |
+
"recommendation": cmp.get("recommendation"),
|
| 96 |
+
"blocker_count": len(cmp.get("blockers") or []),
|
| 97 |
+
})
|
| 98 |
+
|
| 99 |
+
# Also surface LabelEval registry models when present (non-fatal)
|
| 100 |
+
registry = []
|
| 101 |
+
try:
|
| 102 |
+
from sensorflow.evaluation.records import get_store
|
| 103 |
+
for m in get_store().all("models"):
|
| 104 |
+
registry.append({
|
| 105 |
+
"model_id": m.model_id,
|
| 106 |
+
"model_version": m.model_version,
|
| 107 |
+
"metrics": m.metrics.model_dump() if hasattr(m.metrics, "model_dump") else {},
|
| 108 |
+
"regression_status": m.regression_status,
|
| 109 |
+
})
|
| 110 |
+
except Exception:
|
| 111 |
+
pass
|
| 112 |
+
|
| 113 |
+
return {
|
| 114 |
+
"baseline_run_id": baseline.run_id,
|
| 115 |
+
"run_ids": run_ids,
|
| 116 |
+
"models": models,
|
| 117 |
+
"metric_matrix": matrix,
|
| 118 |
+
"pairwise": pairwise,
|
| 119 |
+
"recommendations": recommendations,
|
| 120 |
+
"label_eval_registry": registry,
|
| 121 |
+
"policy": {**analysis.DEFAULT_PROMOTION_POLICY, **(policy or {})},
|
| 122 |
+
}
|
sensorflow/platform/container_quality.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Container quality profiles — wraps MegaEval container table + metric engine."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from typing import Any, Dict, List, Optional
|
| 6 |
+
|
| 7 |
+
import numpy as np
|
| 8 |
+
|
| 9 |
+
from sensorflow.megaeval import population as pop_mod
|
| 10 |
+
from sensorflow.megaeval.population import DIMENSIONS
|
| 11 |
+
from sensorflow.megaeval.runs import CONTAINER_STATUS, EvaluationRun, MegaStore, get_mega_store
|
| 12 |
+
from sensorflow.platform.metrics_engine import container_quality_metrics, aggregate_container_rows
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _enrich_row(r: Dict[str, Any]) -> Dict[str, Any]:
|
| 16 |
+
tp, fp, fn = int(r["tp"]), int(r["fp"]), int(r["fn"])
|
| 17 |
+
sum_iou = float(r.get("sum_iou") or 0.0)
|
| 18 |
+
n = int(r.get("n_objects") or 0)
|
| 19 |
+
verified = int(r.get("verified") or 0)
|
| 20 |
+
reviewed = int(r.get("reviewed") or 0)
|
| 21 |
+
# MegaEval does not yet track disputed/auto_accept/hitl separately;
|
| 22 |
+
# approximate: reviewed-verified ≈ HITL queue residue; rest unverified.
|
| 23 |
+
hitl = max(0, reviewed - verified)
|
| 24 |
+
auto_accepted = 0 # TODO(Phase 2): wire triage auto-accept
|
| 25 |
+
disputed = 0 # TODO(Phase 2): grader disagreement → disputed
|
| 26 |
+
metrics = container_quality_metrics(
|
| 27 |
+
tp=tp, fp=fp, fn=fn, sum_iou=sum_iou, n_objects=n,
|
| 28 |
+
verified=verified, reviewed=reviewed, disputed=disputed,
|
| 29 |
+
auto_accepted=auto_accepted, hitl=hitl,
|
| 30 |
+
anomalies=int(r.get("anomalies") or 0),
|
| 31 |
+
)
|
| 32 |
+
dims = {dim: DIMENSIONS[dim][int(r[dim])] for dim in pop_mod.CONTAINER_DIMS if dim in r}
|
| 33 |
+
return {
|
| 34 |
+
"container_id": int(r["container_id"]),
|
| 35 |
+
"status": CONTAINER_STATUS[int(r["status"])] if "status" in r else None,
|
| 36 |
+
"risk_score": round(float(r.get("risk_score") or 0), 4),
|
| 37 |
+
"dims": dims,
|
| 38 |
+
"drill_down": {
|
| 39 |
+
"object_endpoint": f"/api/megaeval/runs/{{run_id}}/containers/{int(r['container_id'])}/objects",
|
| 40 |
+
"annotation_ids": [], # forensic load on demand
|
| 41 |
+
},
|
| 42 |
+
**metrics,
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def build_container_quality_profile(
|
| 47 |
+
run_id: str,
|
| 48 |
+
*,
|
| 49 |
+
container_id: Optional[int] = None,
|
| 50 |
+
sort: str = "highest_risk",
|
| 51 |
+
limit: int = 50,
|
| 52 |
+
offset: int = 0,
|
| 53 |
+
store: Optional[MegaStore] = None,
|
| 54 |
+
) -> Dict[str, Any]:
|
| 55 |
+
store = store or get_mega_store()
|
| 56 |
+
run = store.runs.get(run_id)
|
| 57 |
+
if run is None:
|
| 58 |
+
raise KeyError(f"Unknown evaluation run {run_id}")
|
| 59 |
+
if run.status != "published":
|
| 60 |
+
raise RuntimeError(f"Run {run_id} is {run.status}; publish before quality profile")
|
| 61 |
+
|
| 62 |
+
art = store.artifacts(run_id)
|
| 63 |
+
df = art["containers"].copy()
|
| 64 |
+
tp, fp, fn = df["tp"], df["fp"], df["fn"]
|
| 65 |
+
with np.errstate(divide="ignore", invalid="ignore"):
|
| 66 |
+
df["recall"] = np.where((tp + fn) > 0, tp / (tp + fn), np.nan)
|
| 67 |
+
df["precision"] = np.where((tp + fp) > 0, tp / (tp + fp), np.nan)
|
| 68 |
+
df["mean_iou"] = np.where(tp > 0, df["sum_iou"] / tp, np.nan)
|
| 69 |
+
|
| 70 |
+
if container_id is not None:
|
| 71 |
+
df = df[df["container_id"] == int(container_id)]
|
| 72 |
+
if df.empty:
|
| 73 |
+
raise KeyError(f"Unknown container {container_id} in run {run_id}")
|
| 74 |
+
|
| 75 |
+
sort_map = {
|
| 76 |
+
"highest_risk": ("risk_score", False),
|
| 77 |
+
"worst_recall": ("recall", True),
|
| 78 |
+
"worst_precision": ("precision", True),
|
| 79 |
+
"worst_iou": ("mean_iou", True),
|
| 80 |
+
"most_anomalies": ("anomalies", False),
|
| 81 |
+
"least_verified": ("verified", True),
|
| 82 |
+
}
|
| 83 |
+
metric, ascending = sort_map.get(sort, sort_map["highest_risk"])
|
| 84 |
+
df = df.sort_values(metric, ascending=ascending, na_position="last")
|
| 85 |
+
total = len(df)
|
| 86 |
+
page = df.iloc[offset: offset + max(1, min(limit, 500))]
|
| 87 |
+
|
| 88 |
+
rows: List[Dict[str, Any]] = []
|
| 89 |
+
for _, series in page.iterrows():
|
| 90 |
+
rows.append(_enrich_row(series.to_dict()))
|
| 91 |
+
|
| 92 |
+
summary = aggregate_container_rows(rows) if rows else container_quality_metrics(
|
| 93 |
+
tp=0, fp=0, fn=0
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
return {
|
| 97 |
+
"run_id": run_id,
|
| 98 |
+
"model_version": run.model_version,
|
| 99 |
+
"population_id": run.population_id,
|
| 100 |
+
"container_id": container_id,
|
| 101 |
+
"total": total,
|
| 102 |
+
"sort": sort,
|
| 103 |
+
"offset": offset,
|
| 104 |
+
"limit": limit,
|
| 105 |
+
"summary": summary,
|
| 106 |
+
"containers": rows,
|
| 107 |
+
"verification_headline": summary.get("verification"),
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def profile_from_counts(
|
| 112 |
+
*,
|
| 113 |
+
container_id: str,
|
| 114 |
+
tp: int,
|
| 115 |
+
fp: int,
|
| 116 |
+
fn: int,
|
| 117 |
+
sum_iou: float = 0.0,
|
| 118 |
+
n_objects: int = 0,
|
| 119 |
+
verified: int = 0,
|
| 120 |
+
reviewed: int = 0,
|
| 121 |
+
disputed: int = 0,
|
| 122 |
+
auto_accepted: int = 0,
|
| 123 |
+
hitl: int = 0,
|
| 124 |
+
anomalies: int = 0,
|
| 125 |
+
) -> Dict[str, Any]:
|
| 126 |
+
"""Unit-testable path without MegaStore."""
|
| 127 |
+
metrics = container_quality_metrics(
|
| 128 |
+
tp=tp, fp=fp, fn=fn, sum_iou=sum_iou, n_objects=n_objects or (tp + fp),
|
| 129 |
+
verified=verified, reviewed=reviewed, disputed=disputed,
|
| 130 |
+
auto_accepted=auto_accepted, hitl=hitl, anomalies=anomalies,
|
| 131 |
+
)
|
| 132 |
+
return {"container_id": container_id, **metrics}
|
sensorflow/platform/entities.py
ADDED
|
@@ -0,0 +1,306 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Platform entity schemas with versioning / provenance.
|
| 2 |
+
|
| 3 |
+
Prefer these as the shared vocabulary across LabelEval + MegaEval.
|
| 4 |
+
Existing store models in evaluation.records remain the persistence layer for
|
| 5 |
+
label QA; MegaEval continues to use npz cubes. These Pydantic models are the
|
| 6 |
+
contract for APIs, evidence packages, and later lakehouse rows.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
|
| 11 |
+
from datetime import datetime, timezone
|
| 12 |
+
from typing import Any, Dict, List, Literal, Optional
|
| 13 |
+
from uuid import uuid4
|
| 14 |
+
|
| 15 |
+
from pydantic import BaseModel, Field
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def _now() -> str:
|
| 19 |
+
return datetime.now(timezone.utc).isoformat()
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def _id(prefix: str) -> str:
|
| 23 |
+
return f"{prefix}-{uuid4().hex[:10]}"
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class Provenance(BaseModel):
|
| 27 |
+
"""Lineage fields required on durable evaluation artifacts."""
|
| 28 |
+
|
| 29 |
+
created_at: str = Field(default_factory=_now)
|
| 30 |
+
created_by: str = "system"
|
| 31 |
+
schema_version: str = "platform-1.0"
|
| 32 |
+
parent_ids: List[str] = Field(default_factory=list)
|
| 33 |
+
source_system: str = "sensorflow" # labeleval | megaeval | pipeline | platform
|
| 34 |
+
seed: Optional[int] = None
|
| 35 |
+
code_version: str = "phase1"
|
| 36 |
+
notes: str = ""
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class VersionedEntity(BaseModel):
|
| 40 |
+
version: str = "v1"
|
| 41 |
+
provenance: Provenance = Field(default_factory=Provenance)
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
# ------------------------------------------------------------------ hierarchy
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class DatasetEntity(VersionedEntity):
|
| 48 |
+
dataset_id: str
|
| 49 |
+
name: str = ""
|
| 50 |
+
population_id: Optional[str] = None # MegaEval link
|
| 51 |
+
num_frames: int = 0
|
| 52 |
+
gt_type: Optional[str] = None
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class Container(VersionedEntity):
|
| 56 |
+
"""Scene/segment holding related objects (MegaEval container_id)."""
|
| 57 |
+
|
| 58 |
+
container_id: str
|
| 59 |
+
dataset_id: Optional[str] = None
|
| 60 |
+
run_id: Optional[str] = None
|
| 61 |
+
mega_container_id: Optional[int] = None
|
| 62 |
+
drive_id: Optional[str] = None
|
| 63 |
+
scene_id: Optional[str] = None
|
| 64 |
+
dims: Dict[str, str] = Field(default_factory=dict)
|
| 65 |
+
n_objects: int = 0
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
class Group(VersionedEntity):
|
| 69 |
+
"""Logical grouping: verification bucket OR cohort alias."""
|
| 70 |
+
|
| 71 |
+
group_id: str
|
| 72 |
+
kind: Literal["verification", "cohort", "custom"] = "custom"
|
| 73 |
+
dataset_id: Optional[str] = None
|
| 74 |
+
member_ids: List[str] = Field(default_factory=list)
|
| 75 |
+
label: str = ""
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
class Drive(VersionedEntity):
|
| 79 |
+
drive_id: str
|
| 80 |
+
dataset_id: Optional[str] = None
|
| 81 |
+
scene_ids: List[str] = Field(default_factory=list)
|
| 82 |
+
container_ids: List[str] = Field(default_factory=list)
|
| 83 |
+
description: str = ""
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class SceneEntity(VersionedEntity):
|
| 87 |
+
scene_id: str
|
| 88 |
+
dataset_id: Optional[str] = None
|
| 89 |
+
drive_id: Optional[str] = None
|
| 90 |
+
sequence_ids: List[str] = Field(default_factory=list)
|
| 91 |
+
description: str = ""
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
class ScenarioEntity(VersionedEntity):
|
| 95 |
+
scenario_id: str
|
| 96 |
+
scenario_type: str
|
| 97 |
+
dataset_id: Optional[str] = None
|
| 98 |
+
frame_ids: List[str] = Field(default_factory=list)
|
| 99 |
+
description: str = ""
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
class FrameEntity(VersionedEntity):
|
| 103 |
+
frame_id: str
|
| 104 |
+
dataset_id: Optional[str] = None
|
| 105 |
+
scene_id: Optional[str] = None
|
| 106 |
+
sequence_id: Optional[str] = None
|
| 107 |
+
container_id: Optional[str] = None
|
| 108 |
+
index: int = 0
|
| 109 |
+
timestamp_us: int = 0
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
class Sensor(VersionedEntity):
|
| 113 |
+
sensor_id: str
|
| 114 |
+
modality: Literal["lidar", "camera", "radar", "ultrasonic", "other"] = "lidar"
|
| 115 |
+
frame_id: Optional[str] = None
|
| 116 |
+
calibration_ref: Optional[str] = None
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
class ObjectEntity(VersionedEntity):
|
| 120 |
+
object_id: str
|
| 121 |
+
class_name: str = "vehicle"
|
| 122 |
+
frame_id: Optional[str] = None
|
| 123 |
+
container_id: Optional[str] = None
|
| 124 |
+
track_id: Optional[str] = None
|
| 125 |
+
bbox_3d: Optional[List[float]] = None
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
class Trajectory(VersionedEntity):
|
| 129 |
+
trajectory_id: str
|
| 130 |
+
track_id: Optional[str] = None
|
| 131 |
+
object_ids: List[str] = Field(default_factory=list)
|
| 132 |
+
frame_ids: List[str] = Field(default_factory=list)
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
class Label(VersionedEntity):
|
| 136 |
+
"""Alias contract for Annotation — hypothesis, never assumed correct."""
|
| 137 |
+
|
| 138 |
+
label_id: str
|
| 139 |
+
annotation_id: Optional[str] = None
|
| 140 |
+
frame_id: Optional[str] = None
|
| 141 |
+
object_id: Optional[str] = None
|
| 142 |
+
class_name: str = ""
|
| 143 |
+
confidence: float = 0.0
|
| 144 |
+
status: str = "PENDING"
|
| 145 |
+
model_version: Optional[str] = None
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
class AnnotationEntity(Label):
|
| 149 |
+
"""Explicit Annotation mirror for API clarity."""
|
| 150 |
+
|
| 151 |
+
pass
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
# ------------------------------------------------------------------ models / eval
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class ModelEntity(VersionedEntity):
|
| 158 |
+
model_id: str
|
| 159 |
+
name: str = ""
|
| 160 |
+
status: str = "ready"
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
class ModelVersion(VersionedEntity):
|
| 164 |
+
model_version_id: str
|
| 165 |
+
model_id: str
|
| 166 |
+
model_version: str
|
| 167 |
+
checkpoint_ref: Optional[str] = None
|
| 168 |
+
trained_on_dataset: Optional[str] = None
|
| 169 |
+
metrics: Dict[str, Any] = Field(default_factory=dict)
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
class Metric(VersionedEntity):
|
| 173 |
+
metric_id: str
|
| 174 |
+
name: str
|
| 175 |
+
value: Optional[float] = None
|
| 176 |
+
unit: str = "ratio"
|
| 177 |
+
higher_is_better: bool = True
|
| 178 |
+
support: Optional[int] = None
|
| 179 |
+
level: str = "container" # AggregateLevel value
|
| 180 |
+
exact: bool = True
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
class Evaluation(VersionedEntity):
|
| 184 |
+
evaluation_id: str
|
| 185 |
+
level: str
|
| 186 |
+
dataset_id: Optional[str] = None
|
| 187 |
+
population_id: Optional[str] = None
|
| 188 |
+
run_id: Optional[str] = None
|
| 189 |
+
model_version: Optional[str] = None
|
| 190 |
+
label_version: Optional[str] = None
|
| 191 |
+
scope_ref: Optional[str] = None # container/cohort/drive id
|
| 192 |
+
metrics: List[Metric] = Field(default_factory=list)
|
| 193 |
+
status: str = "created"
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
class Cohort(VersionedEntity):
|
| 197 |
+
cohort_id: str
|
| 198 |
+
filters: Dict[str, List[str]] = Field(default_factory=dict)
|
| 199 |
+
group_by: List[str] = Field(default_factory=list)
|
| 200 |
+
n: int = 0
|
| 201 |
+
run_id: Optional[str] = None
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
class Embedding(VersionedEntity):
|
| 205 |
+
embedding_id: str
|
| 206 |
+
ref_type: str = "container" # container | object | frame
|
| 207 |
+
ref_id: str
|
| 208 |
+
dim: int = 0
|
| 209 |
+
# TODO(Phase 3): vector payload / external vector-DB handle
|
| 210 |
+
vector_ref: Optional[str] = None
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
class Anomaly(VersionedEntity):
|
| 214 |
+
anomaly_id: str
|
| 215 |
+
annotation_id: Optional[str] = None
|
| 216 |
+
container_id: Optional[str] = None
|
| 217 |
+
score: float = 0.0
|
| 218 |
+
is_anomaly: bool = False
|
| 219 |
+
detector_scores: Dict[str, float] = Field(default_factory=dict)
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
class Conflict(VersionedEntity):
|
| 223 |
+
"""Safety / label conflict placeholder — Phase 4 fills SSAM DRAC/DeltaS."""
|
| 224 |
+
|
| 225 |
+
conflict_id: str
|
| 226 |
+
kind: str = "label" # label | trajectory | ssam_drac | ssam_deltas
|
| 227 |
+
entity_ids: List[str] = Field(default_factory=list)
|
| 228 |
+
severity: str = "medium"
|
| 229 |
+
# TODO(Phase 4): full SSAM conflict engine fields
|
| 230 |
+
detail: Dict[str, Any] = Field(default_factory=dict)
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
class ODDDefinition(VersionedEntity):
|
| 234 |
+
odd_id: str
|
| 235 |
+
name: str
|
| 236 |
+
axes: Dict[str, List[str]] = Field(default_factory=dict)
|
| 237 |
+
# TODO(Phase 5): combinatorial coverage rules
|
| 238 |
+
description: str = ""
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
class ODDObservation(VersionedEntity):
|
| 242 |
+
observation_id: str
|
| 243 |
+
odd_id: str
|
| 244 |
+
observed_axes: Dict[str, str] = Field(default_factory=dict)
|
| 245 |
+
covered: bool = False
|
| 246 |
+
support: int = 0
|
| 247 |
+
run_id: Optional[str] = None
|
| 248 |
+
|
| 249 |
+
|
| 250 |
+
class QualityGateDef(VersionedEntity):
|
| 251 |
+
gate_id: str
|
| 252 |
+
name: str
|
| 253 |
+
gate_type: Literal[
|
| 254 |
+
"scenario", "coverage", "regression", "safety", "release", "quality", "launch"
|
| 255 |
+
]
|
| 256 |
+
thresholds: Dict[str, float] = Field(default_factory=dict)
|
| 257 |
+
config_path: Optional[str] = None
|
| 258 |
+
enabled: bool = True
|
| 259 |
+
|
| 260 |
+
|
| 261 |
+
class GateResult(VersionedEntity):
|
| 262 |
+
result_id: str
|
| 263 |
+
gate_id: str
|
| 264 |
+
gate_type: str
|
| 265 |
+
passed: Optional[bool] = None # None = not yet evaluated (skeleton)
|
| 266 |
+
failures: List[Dict[str, Any]] = Field(default_factory=list)
|
| 267 |
+
metrics: Dict[str, Any] = Field(default_factory=dict)
|
| 268 |
+
thresholds: Dict[str, float] = Field(default_factory=dict)
|
| 269 |
+
scope_ref: Optional[str] = None
|
| 270 |
+
ready: bool = False # False until Phase N wires real checks
|
| 271 |
+
message: str = ""
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
class HumanReviewEntity(VersionedEntity):
|
| 275 |
+
review_id: str
|
| 276 |
+
task_id: Optional[str] = None
|
| 277 |
+
annotation_id: Optional[str] = None
|
| 278 |
+
reviewer: str = "human-reviewer"
|
| 279 |
+
action: Dict[str, Any] = Field(default_factory=dict)
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
class ComputeUsage(VersionedEntity):
|
| 283 |
+
usage_id: str = Field(default_factory=lambda: _id("compute"))
|
| 284 |
+
stage: str = ""
|
| 285 |
+
units: int = 0
|
| 286 |
+
run_id: str = ""
|
| 287 |
+
evaluation_id: Optional[str] = None
|
| 288 |
+
hardware: str = "cpu"
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
class EvidencePackage(VersionedEntity):
|
| 292 |
+
"""Exportable evaluation artifact — Phase 1 stub with real fields when available."""
|
| 293 |
+
|
| 294 |
+
package_id: str = Field(default_factory=lambda: _id("evidence"))
|
| 295 |
+
evaluation_id: str
|
| 296 |
+
dataset_id: Optional[str] = None
|
| 297 |
+
dataset_version: Optional[str] = None
|
| 298 |
+
population_id: Optional[str] = None
|
| 299 |
+
model_versions: List[str] = Field(default_factory=list)
|
| 300 |
+
label_version: Optional[str] = None
|
| 301 |
+
metrics: Dict[str, Any] = Field(default_factory=dict)
|
| 302 |
+
thresholds: Dict[str, Any] = Field(default_factory=dict)
|
| 303 |
+
gate_results: List[Dict[str, Any]] = Field(default_factory=list)
|
| 304 |
+
compute_usage: List[Dict[str, Any]] = Field(default_factory=list)
|
| 305 |
+
drill_down_ids: Dict[str, List[str]] = Field(default_factory=dict)
|
| 306 |
+
placeholders: List[str] = Field(default_factory=list)
|
sensorflow/platform/evidence.py
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evidence package export — JSON stub with real fields when available."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
from typing import Any, Dict, List, Optional
|
| 8 |
+
|
| 9 |
+
from sensorflow.platform.entities import EvidencePackage, Provenance, _id
|
| 10 |
+
|
| 11 |
+
EVIDENCE_DIR = Path("runs/platform/evidence")
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def build_evidence_package(
|
| 15 |
+
*,
|
| 16 |
+
evaluation_id: Optional[str] = None,
|
| 17 |
+
run_id: Optional[str] = None,
|
| 18 |
+
dataset_id: Optional[str] = None,
|
| 19 |
+
sequence_id: Optional[str] = None,
|
| 20 |
+
candidate_run_id: Optional[str] = None,
|
| 21 |
+
baseline_run_id: Optional[str] = None,
|
| 22 |
+
include_gates: bool = True,
|
| 23 |
+
) -> EvidencePackage:
|
| 24 |
+
placeholders: List[str] = []
|
| 25 |
+
metrics: Dict[str, Any] = {}
|
| 26 |
+
model_versions: List[str] = []
|
| 27 |
+
population_id = None
|
| 28 |
+
label_version = None
|
| 29 |
+
dataset_version = None
|
| 30 |
+
compute_usage: List[Dict[str, Any]] = []
|
| 31 |
+
drill: Dict[str, List[str]] = {}
|
| 32 |
+
thresholds: Dict[str, Any] = {}
|
| 33 |
+
gate_results: List[Dict[str, Any]] = []
|
| 34 |
+
|
| 35 |
+
eval_id = evaluation_id or run_id or _id("eval")
|
| 36 |
+
|
| 37 |
+
# MegaEval run headline
|
| 38 |
+
if run_id or candidate_run_id:
|
| 39 |
+
rid = run_id or candidate_run_id
|
| 40 |
+
try:
|
| 41 |
+
from sensorflow.megaeval.runs import get_mega_store
|
| 42 |
+
store = get_mega_store()
|
| 43 |
+
run = store.runs.get(rid) # type: ignore[arg-type]
|
| 44 |
+
if run:
|
| 45 |
+
metrics["headline"] = dict(run.headline or {})
|
| 46 |
+
model_versions.append(run.model_version)
|
| 47 |
+
population_id = run.population_id
|
| 48 |
+
label_version = getattr(run, "label_version", None)
|
| 49 |
+
drill["run_ids"] = [run.run_id]
|
| 50 |
+
else:
|
| 51 |
+
placeholders.append(f"mega_run_missing:{rid}")
|
| 52 |
+
except Exception as exc:
|
| 53 |
+
placeholders.append(f"mega_run_error:{exc}")
|
| 54 |
+
else:
|
| 55 |
+
placeholders.append("run_id")
|
| 56 |
+
|
| 57 |
+
if baseline_run_id:
|
| 58 |
+
try:
|
| 59 |
+
from sensorflow.megaeval.runs import get_mega_store
|
| 60 |
+
br = get_mega_store().runs.get(baseline_run_id)
|
| 61 |
+
if br:
|
| 62 |
+
model_versions.append(br.model_version)
|
| 63 |
+
drill.setdefault("run_ids", []).append(br.run_id)
|
| 64 |
+
except Exception:
|
| 65 |
+
placeholders.append("baseline_run")
|
| 66 |
+
|
| 67 |
+
# LabelEval dataset + process units
|
| 68 |
+
if dataset_id:
|
| 69 |
+
try:
|
| 70 |
+
from sensorflow.evaluation.records import get_store
|
| 71 |
+
from sensorflow.evaluation.process_units import usage_summary
|
| 72 |
+
store = get_store()
|
| 73 |
+
ds = store.get("datasets", dataset_id)
|
| 74 |
+
if ds:
|
| 75 |
+
dataset_version = ds.version
|
| 76 |
+
drill["dataset_ids"] = [dataset_id]
|
| 77 |
+
else:
|
| 78 |
+
placeholders.append(f"dataset_missing:{dataset_id}")
|
| 79 |
+
summary = usage_summary(store)
|
| 80 |
+
compute_usage = [summary] if isinstance(summary, dict) else list(summary or [])
|
| 81 |
+
except Exception as exc:
|
| 82 |
+
placeholders.append(f"labeleval_error:{exc}")
|
| 83 |
+
compute_usage = []
|
| 84 |
+
else:
|
| 85 |
+
placeholders.append("dataset_id")
|
| 86 |
+
|
| 87 |
+
if include_gates:
|
| 88 |
+
try:
|
| 89 |
+
from sensorflow.platform.gates import evaluate_multi_gates, load_gate_config
|
| 90 |
+
cfg = load_gate_config()
|
| 91 |
+
thresholds = {k: v.get("thresholds", {}) for k, v in cfg.get("gates", {}).items()}
|
| 92 |
+
gate_payload = evaluate_multi_gates(
|
| 93 |
+
sequence_id=sequence_id,
|
| 94 |
+
candidate_run_id=candidate_run_id or run_id,
|
| 95 |
+
baseline_run_id=baseline_run_id,
|
| 96 |
+
)
|
| 97 |
+
gate_results = gate_payload.get("gates") or []
|
| 98 |
+
except Exception as exc:
|
| 99 |
+
placeholders.append(f"gates_error:{exc}")
|
| 100 |
+
|
| 101 |
+
if not metrics:
|
| 102 |
+
placeholders.append("metrics")
|
| 103 |
+
if not model_versions:
|
| 104 |
+
placeholders.append("model_versions")
|
| 105 |
+
model_versions = ["placeholder-model"]
|
| 106 |
+
if not compute_usage:
|
| 107 |
+
compute_usage = [{"stage": "platform_evidence", "units": 0, "note": "placeholder"}]
|
| 108 |
+
placeholders.append("compute_usage_detail")
|
| 109 |
+
|
| 110 |
+
return EvidencePackage(
|
| 111 |
+
package_id=_id("evidence"),
|
| 112 |
+
evaluation_id=eval_id,
|
| 113 |
+
dataset_id=dataset_id,
|
| 114 |
+
dataset_version=dataset_version,
|
| 115 |
+
population_id=population_id,
|
| 116 |
+
model_versions=model_versions,
|
| 117 |
+
label_version=label_version,
|
| 118 |
+
metrics=metrics,
|
| 119 |
+
thresholds=thresholds,
|
| 120 |
+
gate_results=gate_results,
|
| 121 |
+
compute_usage=compute_usage,
|
| 122 |
+
drill_down_ids=drill,
|
| 123 |
+
placeholders=sorted(set(placeholders)),
|
| 124 |
+
provenance=Provenance(
|
| 125 |
+
source_system="platform",
|
| 126 |
+
notes="Phase 1 evidence stub — fill placeholders in later phases",
|
| 127 |
+
parent_ids=[x for x in [run_id, dataset_id, sequence_id] if x],
|
| 128 |
+
),
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def export_evidence_package(package: EvidencePackage, path: Optional[Path] = None) -> Path:
|
| 133 |
+
EVIDENCE_DIR.mkdir(parents=True, exist_ok=True)
|
| 134 |
+
path = path or (EVIDENCE_DIR / f"{package.package_id}.json")
|
| 135 |
+
# Append-only: never overwrite an existing package id silently
|
| 136 |
+
if path.exists():
|
| 137 |
+
path = EVIDENCE_DIR / f"{package.package_id}-{_id('rev')}.json"
|
| 138 |
+
with open(path, "w") as f:
|
| 139 |
+
json.dump(package.model_dump(), f, indent=2)
|
| 140 |
+
return path
|
sensorflow/platform/gates.py
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Multi-gate readiness skeleton — config-driven thresholds.
|
| 2 |
+
|
| 3 |
+
Extends existing QualityGate + LaunchGateEvaluator toward:
|
| 4 |
+
Scenario | Coverage | Regression | Safety | Release
|
| 5 |
+
|
| 6 |
+
Phases 4–6 fill Safety/ODD/Release deeply; Phase 1 returns honest ready=False
|
| 7 |
+
for unwired gates and real results for quality/launch when sequence artifacts exist.
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import json
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from typing import Any, Dict, List, Optional
|
| 15 |
+
|
| 16 |
+
from sensorflow.platform.entities import GateResult, QualityGateDef, Provenance, _id
|
| 17 |
+
|
| 18 |
+
DEFAULT_GATE_CONFIG_PATH = Path("runs/platform/gate_config.json")
|
| 19 |
+
|
| 20 |
+
# Config-driven defaults — never hard-code safety cutoffs in callers.
|
| 21 |
+
DEFAULT_GATE_CONFIG: Dict[str, Any] = {
|
| 22 |
+
"schema_version": "platform-gates-1.0",
|
| 23 |
+
"gates": {
|
| 24 |
+
"scenario": {
|
| 25 |
+
"enabled": True,
|
| 26 |
+
"ready": False, # Phase 5/6
|
| 27 |
+
"thresholds": {"min_scenario_types_covered": 5},
|
| 28 |
+
"message": "TODO(Phase 5): scenario coverage vs ODDDefinition",
|
| 29 |
+
},
|
| 30 |
+
"coverage": {
|
| 31 |
+
"enabled": True,
|
| 32 |
+
"ready": False,
|
| 33 |
+
"thresholds": {"min_cohort_support": 200, "min_dim_coverage": 0.8},
|
| 34 |
+
"message": "TODO(Phase 5): ODD combinatorial coverage via MegaEval cube",
|
| 35 |
+
},
|
| 36 |
+
"regression": {
|
| 37 |
+
"enabled": True,
|
| 38 |
+
"ready": True, # can use MegaEval compare / LabelEval regression when run_ids provided
|
| 39 |
+
"thresholds": {
|
| 40 |
+
"max_recall_drop": 0.010,
|
| 41 |
+
"max_precision_drop": 0.015,
|
| 42 |
+
"max_safety_recall_drop": 0.005,
|
| 43 |
+
},
|
| 44 |
+
"message": "Uses MegaEval promotion policy when candidate/baseline runs supplied",
|
| 45 |
+
},
|
| 46 |
+
"safety": {
|
| 47 |
+
"enabled": True,
|
| 48 |
+
"ready": False,
|
| 49 |
+
"thresholds": {}, # filled in Phase 4 from config only
|
| 50 |
+
"message": "TODO(Phase 4): SSAM DRAC/DeltaS conflict engine",
|
| 51 |
+
},
|
| 52 |
+
"release": {
|
| 53 |
+
"enabled": True,
|
| 54 |
+
"ready": False,
|
| 55 |
+
"thresholds": {"require_all_prior_passed": 1.0},
|
| 56 |
+
"message": "TODO(Phase 6): release orchestration after Safety+Regression",
|
| 57 |
+
},
|
| 58 |
+
"quality": {
|
| 59 |
+
"enabled": True,
|
| 60 |
+
"ready": True,
|
| 61 |
+
"thresholds": {
|
| 62 |
+
"map_3d": 0.65,
|
| 63 |
+
"orientation_error_deg": 5.0,
|
| 64 |
+
"id_swap_rate": 0.02,
|
| 65 |
+
"track_fragmentation_rate": 0.05,
|
| 66 |
+
"position_error_m": 2.0,
|
| 67 |
+
},
|
| 68 |
+
"message": "Delegates to sensorflow.quality_gate.QualityGate when sequence_id set",
|
| 69 |
+
},
|
| 70 |
+
"launch": {
|
| 71 |
+
"enabled": True,
|
| 72 |
+
"ready": True,
|
| 73 |
+
"thresholds": {
|
| 74 |
+
"map_3d": 0.65,
|
| 75 |
+
"orientation_error_deg": 5.0,
|
| 76 |
+
"id_swap_rate": 0.02,
|
| 77 |
+
"track_fragmentation_rate": 0.05,
|
| 78 |
+
},
|
| 79 |
+
"message": "Delegates to LaunchGateEvaluator when sequence_id set",
|
| 80 |
+
},
|
| 81 |
+
},
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def load_gate_config(path: Optional[Path] = None) -> Dict[str, Any]:
|
| 86 |
+
path = path or DEFAULT_GATE_CONFIG_PATH
|
| 87 |
+
if path.exists():
|
| 88 |
+
with open(path) as f:
|
| 89 |
+
cfg = json.load(f)
|
| 90 |
+
# merge missing keys from defaults without wiping user thresholds
|
| 91 |
+
merged = json.loads(json.dumps(DEFAULT_GATE_CONFIG))
|
| 92 |
+
for gname, gcfg in (cfg.get("gates") or {}).items():
|
| 93 |
+
if gname in merged["gates"]:
|
| 94 |
+
merged["gates"][gname].update(gcfg)
|
| 95 |
+
else:
|
| 96 |
+
merged["gates"][gname] = gcfg
|
| 97 |
+
return merged
|
| 98 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 99 |
+
with open(path, "w") as f:
|
| 100 |
+
json.dump(DEFAULT_GATE_CONFIG, f, indent=2)
|
| 101 |
+
return json.loads(json.dumps(DEFAULT_GATE_CONFIG))
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def save_gate_config(cfg: Dict[str, Any], path: Optional[Path] = None) -> Path:
|
| 105 |
+
path = path or DEFAULT_GATE_CONFIG_PATH
|
| 106 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 107 |
+
with open(path, "w") as f:
|
| 108 |
+
json.dump(cfg, f, indent=2)
|
| 109 |
+
return path
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
GATE_ORDER = ["scenario", "coverage", "regression", "safety", "quality", "launch", "release"]
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def list_gate_defs(cfg: Optional[Dict[str, Any]] = None) -> List[QualityGateDef]:
|
| 116 |
+
cfg = cfg or load_gate_config()
|
| 117 |
+
out = []
|
| 118 |
+
for name in GATE_ORDER:
|
| 119 |
+
g = cfg["gates"].get(name, {})
|
| 120 |
+
out.append(QualityGateDef(
|
| 121 |
+
gate_id=f"gate-{name}",
|
| 122 |
+
name=name.title(),
|
| 123 |
+
gate_type=name, # type: ignore[arg-type]
|
| 124 |
+
thresholds=dict(g.get("thresholds") or {}),
|
| 125 |
+
config_path=str(DEFAULT_GATE_CONFIG_PATH),
|
| 126 |
+
enabled=bool(g.get("enabled", True)),
|
| 127 |
+
version="v1",
|
| 128 |
+
provenance=Provenance(source_system="platform", notes=g.get("message", "")),
|
| 129 |
+
))
|
| 130 |
+
return out
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def evaluate_multi_gates(
|
| 134 |
+
*,
|
| 135 |
+
sequence_id: Optional[str] = None,
|
| 136 |
+
candidate_run_id: Optional[str] = None,
|
| 137 |
+
baseline_run_id: Optional[str] = None,
|
| 138 |
+
config_path: Optional[Path] = None,
|
| 139 |
+
) -> Dict[str, Any]:
|
| 140 |
+
cfg = load_gate_config(config_path)
|
| 141 |
+
results: List[GateResult] = []
|
| 142 |
+
|
| 143 |
+
for name in GATE_ORDER:
|
| 144 |
+
g = cfg["gates"][name]
|
| 145 |
+
if not g.get("enabled", True):
|
| 146 |
+
continue
|
| 147 |
+
ready = bool(g.get("ready", False))
|
| 148 |
+
thresholds = dict(g.get("thresholds") or {})
|
| 149 |
+
result = GateResult(
|
| 150 |
+
result_id=_id("gateres"),
|
| 151 |
+
gate_id=f"gate-{name}",
|
| 152 |
+
gate_type=name,
|
| 153 |
+
thresholds=thresholds,
|
| 154 |
+
ready=ready,
|
| 155 |
+
message=g.get("message", ""),
|
| 156 |
+
scope_ref=sequence_id or candidate_run_id,
|
| 157 |
+
passed=None,
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
if name == "quality" and sequence_id and ready:
|
| 161 |
+
result = _eval_quality(sequence_id, thresholds, result)
|
| 162 |
+
elif name == "launch" and sequence_id and ready:
|
| 163 |
+
result = _eval_launch(sequence_id, result)
|
| 164 |
+
elif name == "regression" and candidate_run_id and baseline_run_id and ready:
|
| 165 |
+
result = _eval_regression(candidate_run_id, baseline_run_id, thresholds, result)
|
| 166 |
+
elif not ready:
|
| 167 |
+
result.passed = None
|
| 168 |
+
result.message = g.get("message") or f"{name} gate skeleton — not yet wired"
|
| 169 |
+
else:
|
| 170 |
+
result.passed = None
|
| 171 |
+
result.message = f"{name} gate ready but missing required inputs"
|
| 172 |
+
|
| 173 |
+
results.append(result)
|
| 174 |
+
|
| 175 |
+
# Overall readiness: all ready gates that produced a boolean must pass;
|
| 176 |
+
# unwired gates block "release ready" but do not fail the suite silently.
|
| 177 |
+
wired = [r for r in results if r.ready and r.passed is not None]
|
| 178 |
+
unwired = [r for r in results if not r.ready]
|
| 179 |
+
all_wired_passed = all(r.passed for r in wired) if wired else False
|
| 180 |
+
|
| 181 |
+
return {
|
| 182 |
+
"config_path": str(config_path or DEFAULT_GATE_CONFIG_PATH),
|
| 183 |
+
"sequence_id": sequence_id,
|
| 184 |
+
"candidate_run_id": candidate_run_id,
|
| 185 |
+
"baseline_run_id": baseline_run_id,
|
| 186 |
+
"gates": [r.model_dump() for r in results],
|
| 187 |
+
"summary": {
|
| 188 |
+
"wired_count": len(wired),
|
| 189 |
+
"unwired_count": len(unwired),
|
| 190 |
+
"wired_all_passed": all_wired_passed,
|
| 191 |
+
"release_ready": False, # Phase 6
|
| 192 |
+
"unwired_gate_types": [r.gate_type for r in unwired],
|
| 193 |
+
},
|
| 194 |
+
}
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _eval_quality(sequence_id: str, thresholds: Dict[str, float], result: GateResult) -> GateResult:
|
| 198 |
+
try:
|
| 199 |
+
from sensorflow.quality_gate import QualityGate
|
| 200 |
+
from sensorflow.schemas.unified_frame import UnifiedSequence
|
| 201 |
+
|
| 202 |
+
manifest = Path("runs/pipeline") / sequence_id / "manifest.json"
|
| 203 |
+
tracks_path = Path("runs/pipeline") / sequence_id / "tracks.json"
|
| 204 |
+
if not manifest.exists() or not tracks_path.exists():
|
| 205 |
+
result.passed = False
|
| 206 |
+
result.failures = [{"metric": "artifacts", "message": "manifest/tracks missing"}]
|
| 207 |
+
result.message = "Quality gate: run ingest/auto-label/track first"
|
| 208 |
+
return result
|
| 209 |
+
sequence = UnifiedSequence.load(manifest)
|
| 210 |
+
pred_tracks = json.loads(tracks_path.read_text())
|
| 211 |
+
gate = QualityGate(thresholds=thresholds)
|
| 212 |
+
out = gate.evaluate(sequence, pred_tracks)
|
| 213 |
+
result.passed = bool(out.get("passed"))
|
| 214 |
+
result.metrics = out.get("metric_card") or {}
|
| 215 |
+
result.failures = out.get("quality_report", {}).get("failures") or out.get("failures") or []
|
| 216 |
+
result.message = "QualityGate evaluated"
|
| 217 |
+
except Exception as exc:
|
| 218 |
+
result.passed = False
|
| 219 |
+
result.failures = [{"error": str(exc)}]
|
| 220 |
+
result.message = f"Quality gate error: {exc}"
|
| 221 |
+
return result
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def _eval_launch(sequence_id: str, result: GateResult) -> GateResult:
|
| 225 |
+
try:
|
| 226 |
+
from sensorflow.launch_gate_evaluator import LaunchGateEvaluator
|
| 227 |
+
evaluator = LaunchGateEvaluator()
|
| 228 |
+
out = evaluator.evaluate(sequence_id)
|
| 229 |
+
result.passed = bool(out.get("passed"))
|
| 230 |
+
result.failures = out.get("failures") or []
|
| 231 |
+
result.metrics = out.get("metrics") or {}
|
| 232 |
+
result.thresholds = evaluator.thresholds
|
| 233 |
+
result.message = "LaunchGateEvaluator evaluated"
|
| 234 |
+
except Exception as exc:
|
| 235 |
+
result.passed = False
|
| 236 |
+
result.failures = [{"error": str(exc)}]
|
| 237 |
+
result.message = f"Launch gate error: {exc}"
|
| 238 |
+
return result
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def _eval_regression(
|
| 242 |
+
candidate_run_id: str,
|
| 243 |
+
baseline_run_id: str,
|
| 244 |
+
thresholds: Dict[str, float],
|
| 245 |
+
result: GateResult,
|
| 246 |
+
) -> GateResult:
|
| 247 |
+
try:
|
| 248 |
+
from sensorflow.platform.compare import compare_models
|
| 249 |
+
cmp = compare_models(
|
| 250 |
+
[baseline_run_id, candidate_run_id],
|
| 251 |
+
baseline_run_id=baseline_run_id,
|
| 252 |
+
policy=thresholds,
|
| 253 |
+
)
|
| 254 |
+
pair = (cmp.get("pairwise") or [{}])[0]
|
| 255 |
+
rec = pair.get("recommendation")
|
| 256 |
+
result.passed = rec == "PROMOTE"
|
| 257 |
+
result.failures = [{"blocker": b} for b in (pair.get("blockers") or [])]
|
| 258 |
+
result.metrics = {"recommendation": rec, "headline": pair.get("headline")}
|
| 259 |
+
result.message = f"Regression gate: {rec}"
|
| 260 |
+
except Exception as exc:
|
| 261 |
+
result.passed = False
|
| 262 |
+
result.failures = [{"error": str(exc)}]
|
| 263 |
+
result.message = f"Regression gate error: {exc}"
|
| 264 |
+
return result
|
sensorflow/platform/levels.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Evaluation abstraction across aggregate levels.
|
| 2 |
+
|
| 3 |
+
Unifies LabelEval (frame/annotation) and MegaEval (container/cohort/population)
|
| 4 |
+
into one ladder without duplicating storage.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from enum import Enum
|
| 10 |
+
from typing import Any, Dict, List, Optional
|
| 11 |
+
|
| 12 |
+
from pydantic import BaseModel, Field
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class AggregateLevel(str, Enum):
|
| 16 |
+
FRAME = "frame"
|
| 17 |
+
CLIP = "clip" # sequence / short clip
|
| 18 |
+
SCENE = "scene"
|
| 19 |
+
DRIVE = "drive"
|
| 20 |
+
CONTAINER = "container"
|
| 21 |
+
DATASET = "dataset"
|
| 22 |
+
COHORT = "cohort"
|
| 23 |
+
POPULATION = "population"
|
| 24 |
+
|
| 25 |
+
@classmethod
|
| 26 |
+
def ladder(cls) -> List["AggregateLevel"]:
|
| 27 |
+
return [
|
| 28 |
+
cls.FRAME,
|
| 29 |
+
cls.CLIP,
|
| 30 |
+
cls.SCENE,
|
| 31 |
+
cls.DRIVE,
|
| 32 |
+
cls.CONTAINER,
|
| 33 |
+
cls.DATASET,
|
| 34 |
+
cls.COHORT,
|
| 35 |
+
cls.POPULATION,
|
| 36 |
+
]
|
| 37 |
+
|
| 38 |
+
def parent(self) -> Optional["AggregateLevel"]:
|
| 39 |
+
ladder = self.ladder()
|
| 40 |
+
i = ladder.index(self)
|
| 41 |
+
return ladder[i + 1] if i + 1 < len(ladder) else None
|
| 42 |
+
|
| 43 |
+
def child(self) -> Optional["AggregateLevel"]:
|
| 44 |
+
ladder = self.ladder()
|
| 45 |
+
i = ladder.index(self)
|
| 46 |
+
return ladder[i - 1] if i > 0 else None
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# Which backend owns primary aggregates at each level
|
| 50 |
+
LEVEL_BACKEND: Dict[AggregateLevel, str] = {
|
| 51 |
+
AggregateLevel.FRAME: "labeleval",
|
| 52 |
+
AggregateLevel.CLIP: "labeleval",
|
| 53 |
+
AggregateLevel.SCENE: "labeleval",
|
| 54 |
+
AggregateLevel.DRIVE: "platform", # stub until Phase 5/6
|
| 55 |
+
AggregateLevel.CONTAINER: "megaeval",
|
| 56 |
+
AggregateLevel.DATASET: "labeleval",
|
| 57 |
+
AggregateLevel.COHORT: "megaeval",
|
| 58 |
+
AggregateLevel.POPULATION: "megaeval",
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class EvaluationScope(BaseModel):
|
| 63 |
+
"""Points at one aggregate cell without inventing a second DB."""
|
| 64 |
+
|
| 65 |
+
level: AggregateLevel
|
| 66 |
+
evaluation_id: Optional[str] = None
|
| 67 |
+
dataset_id: Optional[str] = None
|
| 68 |
+
population_id: Optional[str] = None
|
| 69 |
+
run_id: Optional[str] = None
|
| 70 |
+
container_id: Optional[str] = None
|
| 71 |
+
mega_container_id: Optional[int] = None
|
| 72 |
+
drive_id: Optional[str] = None
|
| 73 |
+
scene_id: Optional[str] = None
|
| 74 |
+
sequence_id: Optional[str] = None
|
| 75 |
+
frame_id: Optional[str] = None
|
| 76 |
+
cohort_filters: Dict[str, List[str]] = Field(default_factory=dict)
|
| 77 |
+
model_version: Optional[str] = None
|
| 78 |
+
label_version: Optional[str] = None
|
| 79 |
+
|
| 80 |
+
def backend(self) -> str:
|
| 81 |
+
return LEVEL_BACKEND[self.level]
|
| 82 |
+
|
| 83 |
+
def drill_down_hint(self) -> AggregateLevel:
|
| 84 |
+
child = self.level.child()
|
| 85 |
+
return child or self.level
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def summarize_scope(scope: EvaluationScope) -> Dict[str, Any]:
|
| 89 |
+
return {
|
| 90 |
+
"level": scope.level.value,
|
| 91 |
+
"backend": scope.backend(),
|
| 92 |
+
"parent_level": scope.level.parent().value if scope.level.parent() else None,
|
| 93 |
+
"child_level": scope.level.child().value if scope.level.child() else None,
|
| 94 |
+
"refs": {
|
| 95 |
+
k: v
|
| 96 |
+
for k, v in {
|
| 97 |
+
"evaluation_id": scope.evaluation_id,
|
| 98 |
+
"dataset_id": scope.dataset_id,
|
| 99 |
+
"population_id": scope.population_id,
|
| 100 |
+
"run_id": scope.run_id,
|
| 101 |
+
"container_id": scope.container_id,
|
| 102 |
+
"mega_container_id": scope.mega_container_id,
|
| 103 |
+
"drive_id": scope.drive_id,
|
| 104 |
+
"scene_id": scope.scene_id,
|
| 105 |
+
"sequence_id": scope.sequence_id,
|
| 106 |
+
"frame_id": scope.frame_id,
|
| 107 |
+
"model_version": scope.model_version,
|
| 108 |
+
}.items()
|
| 109 |
+
if v is not None
|
| 110 |
+
},
|
| 111 |
+
"cohort_filters": scope.cohort_filters,
|
| 112 |
+
}
|
sensorflow/platform/metrics_engine.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Metric engine: precision/recall/F1/IoU + verification rates.
|
| 2 |
+
|
| 3 |
+
Pure functions — no I/O. Callers feed counts from MegaEval containers or
|
| 4 |
+
LabelEval match stats.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
from typing import Any, Dict, Iterable, List, Optional, Sequence
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def safe_div(num: float, den: float) -> Optional[float]:
|
| 13 |
+
if den <= 0:
|
| 14 |
+
return None
|
| 15 |
+
return num / den
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def round4(v: Optional[float]) -> Optional[float]:
|
| 19 |
+
return None if v is None else round(float(v), 4)
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def precision_recall_f1(tp: int, fp: int, fn: int) -> Dict[str, Optional[float]]:
|
| 23 |
+
precision = safe_div(tp, tp + fp)
|
| 24 |
+
recall = safe_div(tp, tp + fn)
|
| 25 |
+
f1 = None
|
| 26 |
+
if precision is not None and recall is not None and (precision + recall) > 0:
|
| 27 |
+
f1 = 2 * precision * recall / (precision + recall)
|
| 28 |
+
return {
|
| 29 |
+
"precision": round4(precision),
|
| 30 |
+
"recall": round4(recall),
|
| 31 |
+
"f1": round4(f1),
|
| 32 |
+
"tp": tp,
|
| 33 |
+
"fp": fp,
|
| 34 |
+
"fn": fn,
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def mean_iou(sum_iou: float, tp: int) -> Optional[float]:
|
| 39 |
+
return round4(safe_div(sum_iou, tp))
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def verification_rates(
|
| 43 |
+
*,
|
| 44 |
+
n_objects: int,
|
| 45 |
+
verified: int = 0,
|
| 46 |
+
reviewed: int = 0,
|
| 47 |
+
disputed: int = 0,
|
| 48 |
+
auto_accepted: int = 0,
|
| 49 |
+
hitl: int = 0,
|
| 50 |
+
unverified: Optional[int] = None,
|
| 51 |
+
) -> Dict[str, Any]:
|
| 52 |
+
"""Verification / HITL funnel rates at container (or dataset) scope.
|
| 53 |
+
|
| 54 |
+
unverified defaults to n_objects - verified when not provided.
|
| 55 |
+
"""
|
| 56 |
+
n = max(0, int(n_objects))
|
| 57 |
+
verified = max(0, int(verified))
|
| 58 |
+
reviewed = max(0, int(reviewed))
|
| 59 |
+
disputed = max(0, int(disputed))
|
| 60 |
+
auto_accepted = max(0, int(auto_accepted))
|
| 61 |
+
hitl = max(0, int(hitl))
|
| 62 |
+
if unverified is None:
|
| 63 |
+
unverified = max(0, n - verified)
|
| 64 |
+
|
| 65 |
+
def rate(count: int) -> Optional[float]:
|
| 66 |
+
return round4(safe_div(count, n)) if n > 0 else None
|
| 67 |
+
|
| 68 |
+
return {
|
| 69 |
+
"n_objects": n,
|
| 70 |
+
"verified": verified,
|
| 71 |
+
"unverified": unverified,
|
| 72 |
+
"disputed": disputed,
|
| 73 |
+
"auto_accepted": auto_accepted,
|
| 74 |
+
"hitl": hitl,
|
| 75 |
+
"reviewed": reviewed,
|
| 76 |
+
"verified_rate": rate(verified),
|
| 77 |
+
"unverified_rate": rate(unverified),
|
| 78 |
+
"disputed_rate": rate(disputed),
|
| 79 |
+
"auto_accept_rate": rate(auto_accepted),
|
| 80 |
+
"hitl_rate": rate(hitl),
|
| 81 |
+
"review_coverage": rate(reviewed),
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def container_quality_metrics(
|
| 86 |
+
*,
|
| 87 |
+
tp: int,
|
| 88 |
+
fp: int,
|
| 89 |
+
fn: int,
|
| 90 |
+
sum_iou: float = 0.0,
|
| 91 |
+
n_objects: int = 0,
|
| 92 |
+
verified: int = 0,
|
| 93 |
+
reviewed: int = 0,
|
| 94 |
+
disputed: int = 0,
|
| 95 |
+
auto_accepted: int = 0,
|
| 96 |
+
hitl: int = 0,
|
| 97 |
+
anomalies: int = 0,
|
| 98 |
+
) -> Dict[str, Any]:
|
| 99 |
+
prf = precision_recall_f1(tp, fp, fn)
|
| 100 |
+
n = n_objects or (tp + fp)
|
| 101 |
+
ver = verification_rates(
|
| 102 |
+
n_objects=n,
|
| 103 |
+
verified=verified,
|
| 104 |
+
reviewed=reviewed,
|
| 105 |
+
disputed=disputed,
|
| 106 |
+
auto_accepted=auto_accepted,
|
| 107 |
+
hitl=hitl,
|
| 108 |
+
)
|
| 109 |
+
return {
|
| 110 |
+
**prf,
|
| 111 |
+
"mean_iou": mean_iou(sum_iou, tp),
|
| 112 |
+
"anomalies": anomalies,
|
| 113 |
+
"anomaly_rate": round4(safe_div(anomalies, n)) if n else None,
|
| 114 |
+
"verification": ver,
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def aggregate_container_rows(rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]:
|
| 119 |
+
"""Roll up a page of container metric dicts into one profile summary."""
|
| 120 |
+
tp = fp = fn = verified = reviewed = disputed = auto_acc = hitl = anomalies = n = 0
|
| 121 |
+
sum_iou = 0.0
|
| 122 |
+
for r in rows:
|
| 123 |
+
tp += int(r.get("tp") or 0)
|
| 124 |
+
fp += int(r.get("fp") or 0)
|
| 125 |
+
fn += int(r.get("fn") or 0)
|
| 126 |
+
verified += int(r.get("verified") or 0)
|
| 127 |
+
reviewed += int(r.get("reviewed") or 0)
|
| 128 |
+
disputed += int(r.get("disputed") or 0)
|
| 129 |
+
auto_acc += int(r.get("auto_accepted") or 0)
|
| 130 |
+
hitl += int(r.get("hitl") or 0)
|
| 131 |
+
anomalies += int(r.get("anomalies") or 0)
|
| 132 |
+
n += int(r.get("n_objects") or 0)
|
| 133 |
+
mi = r.get("mean_iou")
|
| 134 |
+
tpi = int(r.get("tp") or 0)
|
| 135 |
+
if mi is not None and tpi > 0:
|
| 136 |
+
sum_iou += float(mi) * tpi
|
| 137 |
+
elif r.get("sum_iou") is not None:
|
| 138 |
+
sum_iou += float(r["sum_iou"])
|
| 139 |
+
return container_quality_metrics(
|
| 140 |
+
tp=tp, fp=fp, fn=fn, sum_iou=sum_iou, n_objects=n,
|
| 141 |
+
verified=verified, reviewed=reviewed, disputed=disputed,
|
| 142 |
+
auto_accepted=auto_acc, hitl=hitl, anomalies=anomalies,
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def delta_metrics(
|
| 147 |
+
baseline: Dict[str, Optional[float]],
|
| 148 |
+
candidate: Dict[str, Optional[float]],
|
| 149 |
+
keys: Optional[Iterable[str]] = None,
|
| 150 |
+
) -> List[Dict[str, Any]]:
|
| 151 |
+
keys = list(keys or ("precision", "recall", "f1", "mean_iou", "safety_recall", "anomaly_rate"))
|
| 152 |
+
out = []
|
| 153 |
+
for k in keys:
|
| 154 |
+
b, c = baseline.get(k), candidate.get(k)
|
| 155 |
+
if b is None or c is None:
|
| 156 |
+
continue
|
| 157 |
+
out.append({
|
| 158 |
+
"metric": k,
|
| 159 |
+
"baseline": round4(float(b)),
|
| 160 |
+
"candidate": round4(float(c)),
|
| 161 |
+
"delta": round4(float(c) - float(b)),
|
| 162 |
+
})
|
| 163 |
+
return out
|
src/components/platform/ContainerQualityPanel.tsx
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Container quality cards/table with verification rates (Phase 1 foundation).
|
| 3 |
+
* Extends MegaEval ContainersTab data via /api/containers/quality.
|
| 4 |
+
*/
|
| 5 |
+
import Box from '@mui/material/Box';
|
| 6 |
+
import Table from '@mui/material/Table';
|
| 7 |
+
import TableBody from '@mui/material/TableBody';
|
| 8 |
+
import TableCell from '@mui/material/TableCell';
|
| 9 |
+
import TableHead from '@mui/material/TableHead';
|
| 10 |
+
import TableRow from '@mui/material/TableRow';
|
| 11 |
+
import Typography from '@mui/material/Typography';
|
| 12 |
+
import { useContainerQuality } from '../../services/platform';
|
| 13 |
+
import { ErrorNote, LoadingBox, MetricCard, SectionCard, fmtPct } from '../labeleval/shared';
|
| 14 |
+
|
| 15 |
+
export default function ContainerQualityPanel({
|
| 16 |
+
runId,
|
| 17 |
+
refreshKey = 0,
|
| 18 |
+
}: {
|
| 19 |
+
runId: string | null;
|
| 20 |
+
refreshKey?: number;
|
| 21 |
+
}) {
|
| 22 |
+
const poll = useContainerQuality(runId, refreshKey);
|
| 23 |
+
|
| 24 |
+
if (!runId) {
|
| 25 |
+
return (
|
| 26 |
+
<SectionCard title="Container quality">
|
| 27 |
+
<Typography variant="body2" color="text.secondary">
|
| 28 |
+
Select a published evaluation run to load container quality profiles.
|
| 29 |
+
</Typography>
|
| 30 |
+
</SectionCard>
|
| 31 |
+
);
|
| 32 |
+
}
|
| 33 |
+
if (poll.error && !poll.data) return <ErrorNote error={poll.error} />;
|
| 34 |
+
if (poll.loading && !poll.data) return <LoadingBox label="Loading container quality…" />;
|
| 35 |
+
const data = poll.data;
|
| 36 |
+
if (!data) return null;
|
| 37 |
+
|
| 38 |
+
const v = data.verification_headline || data.summary?.verification;
|
| 39 |
+
|
| 40 |
+
return (
|
| 41 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 42 |
+
<SectionCard
|
| 43 |
+
title="Container quality profile"
|
| 44 |
+
action={
|
| 45 |
+
<Typography variant="caption" color="text.secondary">
|
| 46 |
+
{data.total} containers · {data.model_version}
|
| 47 |
+
</Typography>
|
| 48 |
+
}
|
| 49 |
+
>
|
| 50 |
+
<Box sx={{ display: 'flex', gap: 1.5, flexWrap: 'wrap', mb: 2 }}>
|
| 51 |
+
<MetricCard label="Precision" value={fmtPct(data.summary.precision)} />
|
| 52 |
+
<MetricCard label="Recall" value={fmtPct(data.summary.recall)} />
|
| 53 |
+
<MetricCard label="F1" value={fmtPct(data.summary.f1)} />
|
| 54 |
+
<MetricCard label="Mean IoU" value={fmtPct(data.summary.mean_iou)} />
|
| 55 |
+
<MetricCard label="Verified" value={fmtPct(v?.verified_rate)} sub={`${v?.verified ?? 0} labels`} />
|
| 56 |
+
<MetricCard label="HITL rate" value={fmtPct(v?.hitl_rate)} />
|
| 57 |
+
<MetricCard label="Unverified" value={fmtPct(v?.unverified_rate)} />
|
| 58 |
+
<MetricCard label="Disputed" value={fmtPct(v?.disputed_rate)} sub="Phase 2 hook" />
|
| 59 |
+
</Box>
|
| 60 |
+
|
| 61 |
+
<Table size="small">
|
| 62 |
+
<TableHead>
|
| 63 |
+
<TableRow>
|
| 64 |
+
<TableCell>Container</TableCell>
|
| 65 |
+
<TableCell>Status</TableCell>
|
| 66 |
+
<TableCell align="right">P</TableCell>
|
| 67 |
+
<TableCell align="right">R</TableCell>
|
| 68 |
+
<TableCell align="right">F1</TableCell>
|
| 69 |
+
<TableCell align="right">IoU</TableCell>
|
| 70 |
+
<TableCell align="right">Verified</TableCell>
|
| 71 |
+
<TableCell align="right">HITL</TableCell>
|
| 72 |
+
<TableCell align="right">Risk</TableCell>
|
| 73 |
+
</TableRow>
|
| 74 |
+
</TableHead>
|
| 75 |
+
<TableBody>
|
| 76 |
+
{data.containers.slice(0, 20).map((c) => (
|
| 77 |
+
<TableRow key={c.container_id} hover>
|
| 78 |
+
<TableCell>#{c.container_id}</TableCell>
|
| 79 |
+
<TableCell>{c.status ?? '—'}</TableCell>
|
| 80 |
+
<TableCell align="right">{fmtPct(c.precision)}</TableCell>
|
| 81 |
+
<TableCell align="right">{fmtPct(c.recall)}</TableCell>
|
| 82 |
+
<TableCell align="right">{fmtPct(c.f1)}</TableCell>
|
| 83 |
+
<TableCell align="right">{fmtPct(c.mean_iou)}</TableCell>
|
| 84 |
+
<TableCell align="right">{fmtPct(c.verification?.verified_rate)}</TableCell>
|
| 85 |
+
<TableCell align="right">{fmtPct(c.verification?.hitl_rate)}</TableCell>
|
| 86 |
+
<TableCell align="right">{c.risk_score.toFixed(2)}</TableCell>
|
| 87 |
+
</TableRow>
|
| 88 |
+
))}
|
| 89 |
+
</TableBody>
|
| 90 |
+
</Table>
|
| 91 |
+
</SectionCard>
|
| 92 |
+
</Box>
|
| 93 |
+
);
|
| 94 |
+
}
|
src/components/platform/GateStatusPanel.tsx
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Multi-gate readiness skeleton: Scenario / Coverage / Regression / Safety / Release
|
| 3 |
+
* (+ quality / launch when wired).
|
| 4 |
+
*/
|
| 5 |
+
import Box from '@mui/material/Box';
|
| 6 |
+
import Chip from '@mui/material/Chip';
|
| 7 |
+
import Typography from '@mui/material/Typography';
|
| 8 |
+
import { CheckCircle2, CircleDashed, XCircle } from 'lucide-react';
|
| 9 |
+
import { useGateStatus } from '../../services/platform';
|
| 10 |
+
import { ErrorNote, LoadingBox, SectionCard } from '../labeleval/shared';
|
| 11 |
+
|
| 12 |
+
const DISPLAY_ORDER = ['scenario', 'coverage', 'regression', 'safety', 'quality', 'launch', 'release'];
|
| 13 |
+
|
| 14 |
+
function statusChip(passed: boolean | null, ready: boolean) {
|
| 15 |
+
if (!ready) {
|
| 16 |
+
return <Chip size="small" icon={<CircleDashed size={14} />} label="Skeleton" variant="outlined" />;
|
| 17 |
+
}
|
| 18 |
+
if (passed === true) {
|
| 19 |
+
return <Chip size="small" color="success" icon={<CheckCircle2 size={14} />} label="Pass" />;
|
| 20 |
+
}
|
| 21 |
+
if (passed === false) {
|
| 22 |
+
return <Chip size="small" color="error" icon={<XCircle size={14} />} label="Fail" />;
|
| 23 |
+
}
|
| 24 |
+
return <Chip size="small" label="Pending inputs" variant="outlined" />;
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
export default function GateStatusPanel({
|
| 28 |
+
candidateRunId,
|
| 29 |
+
baselineRunId,
|
| 30 |
+
refreshKey = 0,
|
| 31 |
+
}: {
|
| 32 |
+
candidateRunId: string | null;
|
| 33 |
+
baselineRunId: string | null;
|
| 34 |
+
refreshKey?: number;
|
| 35 |
+
}) {
|
| 36 |
+
const poll = useGateStatus(candidateRunId, baselineRunId, refreshKey);
|
| 37 |
+
|
| 38 |
+
if (poll.error && !poll.data) return <ErrorNote error={poll.error} />;
|
| 39 |
+
if (poll.loading && !poll.data) return <LoadingBox label="Loading gate status…" />;
|
| 40 |
+
const data = poll.data;
|
| 41 |
+
if (!data) return null;
|
| 42 |
+
|
| 43 |
+
const byType = Object.fromEntries(data.gates.map((g) => [g.gate_type, g]));
|
| 44 |
+
|
| 45 |
+
return (
|
| 46 |
+
<SectionCard
|
| 47 |
+
title="Multi-gate readiness"
|
| 48 |
+
action={
|
| 49 |
+
<Typography variant="caption" color="text.secondary">
|
| 50 |
+
{data.summary.release_ready
|
| 51 |
+
? 'Release ready'
|
| 52 |
+
: `${data.summary.unwired_count} gates still Phase 2–6 stubs`}
|
| 53 |
+
</Typography>
|
| 54 |
+
}
|
| 55 |
+
>
|
| 56 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 1 }}>
|
| 57 |
+
{DISPLAY_ORDER.map((type) => {
|
| 58 |
+
const g = byType[type];
|
| 59 |
+
if (!g) return null;
|
| 60 |
+
return (
|
| 61 |
+
<Box
|
| 62 |
+
key={type}
|
| 63 |
+
sx={{
|
| 64 |
+
display: 'flex',
|
| 65 |
+
alignItems: 'center',
|
| 66 |
+
gap: 1.5,
|
| 67 |
+
py: 0.75,
|
| 68 |
+
borderBottom: '1px solid #232a31',
|
| 69 |
+
}}
|
| 70 |
+
>
|
| 71 |
+
<Typography sx={{ minWidth: 110, fontWeight: 600, textTransform: 'capitalize' }}>
|
| 72 |
+
{type}
|
| 73 |
+
</Typography>
|
| 74 |
+
{statusChip(g.passed, g.ready)}
|
| 75 |
+
<Typography variant="body2" color="text.secondary" sx={{ flex: 1 }}>
|
| 76 |
+
{g.message}
|
| 77 |
+
</Typography>
|
| 78 |
+
</Box>
|
| 79 |
+
);
|
| 80 |
+
})}
|
| 81 |
+
</Box>
|
| 82 |
+
</SectionCard>
|
| 83 |
+
);
|
| 84 |
+
}
|
src/components/platform/ModelComparePanel.tsx
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Multi-model compare panel (A vs B vs C) — wraps /api/models/compare.
|
| 3 |
+
*/
|
| 4 |
+
import { useMemo } from 'react';
|
| 5 |
+
import Alert from '@mui/material/Alert';
|
| 6 |
+
import Box from '@mui/material/Box';
|
| 7 |
+
import Table from '@mui/material/Table';
|
| 8 |
+
import TableBody from '@mui/material/TableBody';
|
| 9 |
+
import TableCell from '@mui/material/TableCell';
|
| 10 |
+
import TableHead from '@mui/material/TableHead';
|
| 11 |
+
import TableRow from '@mui/material/TableRow';
|
| 12 |
+
import Typography from '@mui/material/Typography';
|
| 13 |
+
import { compareModels } from '../../services/platform';
|
| 14 |
+
import { usePoll } from '../../services/labeleval';
|
| 15 |
+
import { ErrorNote, LoadingBox, SectionCard, fmtPct } from '../labeleval/shared';
|
| 16 |
+
|
| 17 |
+
export default function ModelComparePanel({
|
| 18 |
+
runId,
|
| 19 |
+
baselineRunId,
|
| 20 |
+
extraRunIds = [],
|
| 21 |
+
refreshKey = 0,
|
| 22 |
+
}: {
|
| 23 |
+
runId: string | null;
|
| 24 |
+
baselineRunId: string | null;
|
| 25 |
+
extraRunIds?: string[];
|
| 26 |
+
refreshKey?: number;
|
| 27 |
+
}) {
|
| 28 |
+
const runIds = useMemo(() => {
|
| 29 |
+
const ids = [baselineRunId, runId, ...extraRunIds].filter(Boolean) as string[];
|
| 30 |
+
return [...new Set(ids)];
|
| 31 |
+
}, [runId, baselineRunId, extraRunIds]);
|
| 32 |
+
|
| 33 |
+
const cmp = usePoll(
|
| 34 |
+
() =>
|
| 35 |
+
runIds.length >= 2
|
| 36 |
+
? compareModels(runIds, baselineRunId)
|
| 37 |
+
: Promise.resolve(null),
|
| 38 |
+
null,
|
| 39 |
+
[runIds.join(','), baselineRunId, refreshKey]
|
| 40 |
+
);
|
| 41 |
+
|
| 42 |
+
if (runIds.length < 2) {
|
| 43 |
+
return (
|
| 44 |
+
<Alert severity="info" variant="outlined">
|
| 45 |
+
Select a baseline run (and optionally more candidates) to compare models A / B / C.
|
| 46 |
+
</Alert>
|
| 47 |
+
);
|
| 48 |
+
}
|
| 49 |
+
if (cmp.error && !cmp.data) return <ErrorNote error={cmp.error} />;
|
| 50 |
+
if (cmp.loading && !cmp.data) return <LoadingBox label="Comparing models…" />;
|
| 51 |
+
const data = cmp.data;
|
| 52 |
+
if (!data) return null;
|
| 53 |
+
|
| 54 |
+
const modelNames = data.models.map((m) => m.model_version);
|
| 55 |
+
|
| 56 |
+
return (
|
| 57 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 58 |
+
<SectionCard
|
| 59 |
+
title="Model compare (A / B / C)"
|
| 60 |
+
action={
|
| 61 |
+
<Typography variant="caption" color="text.secondary">
|
| 62 |
+
Baseline {data.baseline_run_id.slice(0, 8)}…
|
| 63 |
+
</Typography>
|
| 64 |
+
}
|
| 65 |
+
>
|
| 66 |
+
<Box sx={{ display: 'flex', gap: 1, flexWrap: 'wrap', mb: 2 }}>
|
| 67 |
+
{data.recommendations.map((r) => (
|
| 68 |
+
<Alert
|
| 69 |
+
key={r.candidate}
|
| 70 |
+
severity={r.recommendation === 'PROMOTE' ? 'success' : 'warning'}
|
| 71 |
+
variant="outlined"
|
| 72 |
+
sx={{ py: 0 }}
|
| 73 |
+
>
|
| 74 |
+
{r.candidate}: {r.recommendation}
|
| 75 |
+
{r.blocker_count ? ` (${r.blocker_count} blockers)` : ''}
|
| 76 |
+
</Alert>
|
| 77 |
+
))}
|
| 78 |
+
</Box>
|
| 79 |
+
|
| 80 |
+
<Table size="small">
|
| 81 |
+
<TableHead>
|
| 82 |
+
<TableRow>
|
| 83 |
+
<TableCell>Metric</TableCell>
|
| 84 |
+
{modelNames.map((m) => (
|
| 85 |
+
<TableCell key={m} align="right">
|
| 86 |
+
{m}
|
| 87 |
+
</TableCell>
|
| 88 |
+
))}
|
| 89 |
+
</TableRow>
|
| 90 |
+
</TableHead>
|
| 91 |
+
<TableBody>
|
| 92 |
+
{data.metric_matrix.map((row) => (
|
| 93 |
+
<TableRow key={row.metric}>
|
| 94 |
+
<TableCell>{row.metric}</TableCell>
|
| 95 |
+
{modelNames.map((m) => {
|
| 96 |
+
const v = row[m] as number | null | undefined;
|
| 97 |
+
const d = row.deltas_vs_baseline?.[m];
|
| 98 |
+
return (
|
| 99 |
+
<TableCell key={m} align="right">
|
| 100 |
+
{fmtPct(v ?? null)}
|
| 101 |
+
{d !== undefined && d !== null ? (
|
| 102 |
+
<Typography component="span" variant="caption" sx={{ ml: 0.5, color: d < 0 ? '#c62828' : '#2e7d32' }}>
|
| 103 |
+
{d >= 0 ? '+' : ''}
|
| 104 |
+
{(d * 100).toFixed(1)}pp
|
| 105 |
+
</Typography>
|
| 106 |
+
) : null}
|
| 107 |
+
</TableCell>
|
| 108 |
+
);
|
| 109 |
+
})}
|
| 110 |
+
</TableRow>
|
| 111 |
+
))}
|
| 112 |
+
</TableBody>
|
| 113 |
+
</Table>
|
| 114 |
+
</SectionCard>
|
| 115 |
+
</Box>
|
| 116 |
+
);
|
| 117 |
+
}
|
src/pages/CommandCenterPage.tsx
CHANGED
|
@@ -40,6 +40,9 @@ import CompareTab from '../components/megaeval/CompareTab';
|
|
| 40 |
import ReviewTab from '../components/megaeval/ReviewTab';
|
| 41 |
import ShiftTab from '../components/megaeval/ShiftTab';
|
| 42 |
import LineageCard from '../components/megaeval/LineageCard';
|
|
|
|
|
|
|
|
|
|
| 43 |
|
| 44 |
const ACTIVE_STATUSES: RunStatus[] = ['created', 'queued', 'running', 'reducing', 'materializing'];
|
| 45 |
|
|
@@ -49,6 +52,7 @@ type TabId =
|
|
| 49 |
| 'containers'
|
| 50 |
| 'investigation'
|
| 51 |
| 'compare'
|
|
|
|
| 52 |
| 'review'
|
| 53 |
| 'shift'
|
| 54 |
| 'lineage';
|
|
@@ -290,28 +294,49 @@ export default function CommandCenterPage() {
|
|
| 290 |
<Tab label="Containers" value="containers" />
|
| 291 |
<Tab label="Investigation" value="investigation" />
|
| 292 |
<Tab label="Compare" value="compare" />
|
|
|
|
| 293 |
<Tab label="Review" value="review" />
|
| 294 |
<Tab label="Shift" value="shift" />
|
| 295 |
<Tab label="Lineage" value="lineage" />
|
| 296 |
</Tabs>
|
| 297 |
|
| 298 |
{tab === 'quality' ? (
|
| 299 |
-
<
|
|
|
|
|
|
|
|
|
|
| 300 |
) : null}
|
| 301 |
{tab === 'cohorts' ? <CohortTab runId={selectedRun.run_id} refreshKey={publishTick} /> : null}
|
| 302 |
{tab === 'containers' ? (
|
| 303 |
-
<
|
| 304 |
-
runId={selectedRun.run_id}
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
|
|
|
|
|
|
|
|
|
| 309 |
) : null}
|
| 310 |
{tab === 'investigation' ? (
|
| 311 |
<InvestigationTab runId={selectedRun.run_id} onOpenContainer={openContainer} />
|
| 312 |
) : null}
|
| 313 |
{tab === 'compare' ? (
|
| 314 |
-
<
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 315 |
) : null}
|
| 316 |
{tab === 'review' ? (
|
| 317 |
<ReviewTab runId={selectedRun.run_id} review={review} onReview={setReview} />
|
|
|
|
| 40 |
import ReviewTab from '../components/megaeval/ReviewTab';
|
| 41 |
import ShiftTab from '../components/megaeval/ShiftTab';
|
| 42 |
import LineageCard from '../components/megaeval/LineageCard';
|
| 43 |
+
import ContainerQualityPanel from '../components/platform/ContainerQualityPanel';
|
| 44 |
+
import ModelComparePanel from '../components/platform/ModelComparePanel';
|
| 45 |
+
import GateStatusPanel from '../components/platform/GateStatusPanel';
|
| 46 |
|
| 47 |
const ACTIVE_STATUSES: RunStatus[] = ['created', 'queued', 'running', 'reducing', 'materializing'];
|
| 48 |
|
|
|
|
| 52 |
| 'containers'
|
| 53 |
| 'investigation'
|
| 54 |
| 'compare'
|
| 55 |
+
| 'gates'
|
| 56 |
| 'review'
|
| 57 |
| 'shift'
|
| 58 |
| 'lineage';
|
|
|
|
| 294 |
<Tab label="Containers" value="containers" />
|
| 295 |
<Tab label="Investigation" value="investigation" />
|
| 296 |
<Tab label="Compare" value="compare" />
|
| 297 |
+
<Tab label="Gates" value="gates" />
|
| 298 |
<Tab label="Review" value="review" />
|
| 299 |
<Tab label="Shift" value="shift" />
|
| 300 |
<Tab label="Lineage" value="lineage" />
|
| 301 |
</Tabs>
|
| 302 |
|
| 303 |
{tab === 'quality' ? (
|
| 304 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 305 |
+
<QualityTab runId={selectedRun.run_id} runs={runs} refreshKey={publishTick} />
|
| 306 |
+
<ContainerQualityPanel runId={selectedRun.run_id} refreshKey={publishTick} />
|
| 307 |
+
</Box>
|
| 308 |
) : null}
|
| 309 |
{tab === 'cohorts' ? <CohortTab runId={selectedRun.run_id} refreshKey={publishTick} /> : null}
|
| 310 |
{tab === 'containers' ? (
|
| 311 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 312 |
+
<ContainerQualityPanel runId={selectedRun.run_id} refreshKey={publishTick} />
|
| 313 |
+
<ContainersTab
|
| 314 |
+
runId={selectedRun.run_id}
|
| 315 |
+
refreshKey={publishTick}
|
| 316 |
+
drillContainerId={drillContainerId}
|
| 317 |
+
onDrill={setDrillContainerId}
|
| 318 |
+
/>
|
| 319 |
+
</Box>
|
| 320 |
) : null}
|
| 321 |
{tab === 'investigation' ? (
|
| 322 |
<InvestigationTab runId={selectedRun.run_id} onOpenContainer={openContainer} />
|
| 323 |
) : null}
|
| 324 |
{tab === 'compare' ? (
|
| 325 |
+
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 326 |
+
<ModelComparePanel
|
| 327 |
+
runId={selectedRun.run_id}
|
| 328 |
+
baselineRunId={baselineRunId}
|
| 329 |
+
refreshKey={publishTick}
|
| 330 |
+
/>
|
| 331 |
+
<CompareTab runId={selectedRun.run_id} baselineRunId={baselineRunId} refreshKey={publishTick} />
|
| 332 |
+
</Box>
|
| 333 |
+
) : null}
|
| 334 |
+
{tab === 'gates' ? (
|
| 335 |
+
<GateStatusPanel
|
| 336 |
+
candidateRunId={selectedRun.run_id}
|
| 337 |
+
baselineRunId={baselineRunId}
|
| 338 |
+
refreshKey={publishTick}
|
| 339 |
+
/>
|
| 340 |
) : null}
|
| 341 |
{tab === 'review' ? (
|
| 342 |
<ReviewTab runId={selectedRun.run_id} review={review} onReview={setReview} />
|
src/pages/EvaluationPage.tsx
CHANGED
|
@@ -20,6 +20,7 @@ import {
|
|
| 20 |
import { useLabelEval } from '../context/LabelEvalContext';
|
| 21 |
import EvidencePanel from '../components/labeleval/EvidencePanel';
|
| 22 |
import CopilotDrawer from '../components/labeleval/CopilotDrawer';
|
|
|
|
| 23 |
import { SectionCard, StatusChip, LoadingBox, ErrorNote, fmtNum, fmtPct } from '../components/labeleval/shared';
|
| 24 |
|
| 25 |
const PAGE_SIZE = 15;
|
|
@@ -90,6 +91,7 @@ export default function EvaluationPage() {
|
|
| 90 |
|
| 91 |
return (
|
| 92 |
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
|
|
|
| 93 |
<SectionCard title="Evaluation Record Explorer">
|
| 94 |
<Box sx={{ display: 'flex', alignItems: 'center', gap: 2, mb: 2 }}>
|
| 95 |
<Typography variant="body2" sx={{ color: '#8a949e' }}>
|
|
|
|
| 20 |
import { useLabelEval } from '../context/LabelEvalContext';
|
| 21 |
import EvidencePanel from '../components/labeleval/EvidencePanel';
|
| 22 |
import CopilotDrawer from '../components/labeleval/CopilotDrawer';
|
| 23 |
+
import GateStatusPanel from '../components/platform/GateStatusPanel';
|
| 24 |
import { SectionCard, StatusChip, LoadingBox, ErrorNote, fmtNum, fmtPct } from '../components/labeleval/shared';
|
| 25 |
|
| 26 |
const PAGE_SIZE = 15;
|
|
|
|
| 91 |
|
| 92 |
return (
|
| 93 |
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
|
| 94 |
+
<GateStatusPanel candidateRunId={null} baselineRunId={null} />
|
| 95 |
<SectionCard title="Evaluation Record Explorer">
|
| 96 |
<Box sx={{ display: 'flex', alignItems: 'center', gap: 2, mb: 2 }}>
|
| 97 |
<Typography variant="body2" sx={{ color: '#8a949e' }}>
|
src/pages/QualityEnginePage.tsx
CHANGED
|
@@ -5,6 +5,7 @@ import LabelEvaluationEngine from '../components/labeleval/LabelEvaluationEngine
|
|
| 5 |
import StrictQualityValidation from '../components/labeleval/StrictQualityValidation';
|
| 6 |
import GroundTruthComparison from '../components/labeleval/GroundTruthComparison';
|
| 7 |
import GraderDisagreement from '../components/labeleval/GraderDisagreement';
|
|
|
|
| 8 |
import { LoadingBox, ErrorNote } from '../components/labeleval/shared';
|
| 9 |
|
| 10 |
export default function QualityEnginePage() {
|
|
@@ -20,6 +21,7 @@ export default function QualityEnginePage() {
|
|
| 20 |
{metrics.error && !metrics.data ? <ErrorNote error={metrics.error} /> : null}
|
| 21 |
|
| 22 |
<LabelEvaluationEngine pipeline={pipeline ?? null} />
|
|
|
|
| 23 |
<StrictQualityValidation metrics={metrics.data} />
|
| 24 |
<GroundTruthComparison metrics={metrics.data} />
|
| 25 |
<GraderDisagreement metrics={metrics.data} />
|
|
|
|
| 5 |
import StrictQualityValidation from '../components/labeleval/StrictQualityValidation';
|
| 6 |
import GroundTruthComparison from '../components/labeleval/GroundTruthComparison';
|
| 7 |
import GraderDisagreement from '../components/labeleval/GraderDisagreement';
|
| 8 |
+
import GateStatusPanel from '../components/platform/GateStatusPanel';
|
| 9 |
import { LoadingBox, ErrorNote } from '../components/labeleval/shared';
|
| 10 |
|
| 11 |
export default function QualityEnginePage() {
|
|
|
|
| 21 |
{metrics.error && !metrics.data ? <ErrorNote error={metrics.error} /> : null}
|
| 22 |
|
| 23 |
<LabelEvaluationEngine pipeline={pipeline ?? null} />
|
| 24 |
+
<GateStatusPanel candidateRunId={null} baselineRunId={null} />
|
| 25 |
<StrictQualityValidation metrics={metrics.data} />
|
| 26 |
<GroundTruthComparison metrics={metrics.data} />
|
| 27 |
<GraderDisagreement metrics={metrics.data} />
|
src/services/platform.ts
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Platform Phase 1 API client — container quality, multi-model compare, gates, evidence.
|
| 3 |
+
*/
|
| 4 |
+
import { usePoll } from './labeleval';
|
| 5 |
+
|
| 6 |
+
async function handle<T>(res: Response): Promise<T> {
|
| 7 |
+
if (!res.ok) {
|
| 8 |
+
const body = await res.text().catch(() => '');
|
| 9 |
+
throw new Error(`API ${res.status}: ${body || res.statusText}`);
|
| 10 |
+
}
|
| 11 |
+
return res.json() as Promise<T>;
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
function get<T>(url: string): Promise<T> {
|
| 15 |
+
return fetch(url).then((res) => handle<T>(res));
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
function post<T>(url: string, body?: unknown): Promise<T> {
|
| 19 |
+
return fetch(url, {
|
| 20 |
+
method: 'POST',
|
| 21 |
+
headers: { 'Content-Type': 'application/json' },
|
| 22 |
+
body: body === undefined ? undefined : JSON.stringify(body),
|
| 23 |
+
}).then((res) => handle<T>(res));
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
export type VerificationRates = {
|
| 27 |
+
n_objects: number;
|
| 28 |
+
verified: number;
|
| 29 |
+
unverified: number;
|
| 30 |
+
disputed: number;
|
| 31 |
+
auto_accepted: number;
|
| 32 |
+
hitl: number;
|
| 33 |
+
reviewed: number;
|
| 34 |
+
verified_rate: number | null;
|
| 35 |
+
unverified_rate: number | null;
|
| 36 |
+
disputed_rate: number | null;
|
| 37 |
+
auto_accept_rate: number | null;
|
| 38 |
+
hitl_rate: number | null;
|
| 39 |
+
review_coverage: number | null;
|
| 40 |
+
};
|
| 41 |
+
|
| 42 |
+
export type ContainerQualityRow = {
|
| 43 |
+
container_id: number;
|
| 44 |
+
status: string | null;
|
| 45 |
+
risk_score: number;
|
| 46 |
+
precision: number | null;
|
| 47 |
+
recall: number | null;
|
| 48 |
+
f1: number | null;
|
| 49 |
+
mean_iou: number | null;
|
| 50 |
+
verification: VerificationRates;
|
| 51 |
+
dims: Record<string, string>;
|
| 52 |
+
};
|
| 53 |
+
|
| 54 |
+
export type ContainerQualityProfile = {
|
| 55 |
+
run_id: string;
|
| 56 |
+
model_version: string;
|
| 57 |
+
total: number;
|
| 58 |
+
summary: {
|
| 59 |
+
precision: number | null;
|
| 60 |
+
recall: number | null;
|
| 61 |
+
f1: number | null;
|
| 62 |
+
mean_iou: number | null;
|
| 63 |
+
verification: VerificationRates;
|
| 64 |
+
};
|
| 65 |
+
containers: ContainerQualityRow[];
|
| 66 |
+
verification_headline: VerificationRates;
|
| 67 |
+
};
|
| 68 |
+
|
| 69 |
+
export type GateResultRow = {
|
| 70 |
+
result_id: string;
|
| 71 |
+
gate_id: string;
|
| 72 |
+
gate_type: string;
|
| 73 |
+
passed: boolean | null;
|
| 74 |
+
ready: boolean;
|
| 75 |
+
message: string;
|
| 76 |
+
failures: Array<Record<string, unknown>>;
|
| 77 |
+
thresholds: Record<string, number>;
|
| 78 |
+
};
|
| 79 |
+
|
| 80 |
+
export type GateStatusResponse = {
|
| 81 |
+
gates: GateResultRow[];
|
| 82 |
+
summary: {
|
| 83 |
+
wired_count: number;
|
| 84 |
+
unwired_count: number;
|
| 85 |
+
wired_all_passed: boolean;
|
| 86 |
+
release_ready: boolean;
|
| 87 |
+
unwired_gate_types: string[];
|
| 88 |
+
};
|
| 89 |
+
};
|
| 90 |
+
|
| 91 |
+
export type ModelCompareResponse = {
|
| 92 |
+
baseline_run_id: string;
|
| 93 |
+
run_ids: string[];
|
| 94 |
+
models: Array<{
|
| 95 |
+
run_id: string;
|
| 96 |
+
model_version: string;
|
| 97 |
+
role: string;
|
| 98 |
+
headline: Record<string, number | null>;
|
| 99 |
+
}>;
|
| 100 |
+
metric_matrix: Array<{
|
| 101 |
+
metric: string;
|
| 102 |
+
deltas_vs_baseline: Record<string, number | null>;
|
| 103 |
+
[model: string]: unknown;
|
| 104 |
+
}>;
|
| 105 |
+
pairwise: Array<{
|
| 106 |
+
candidate_model: string;
|
| 107 |
+
recommendation: string;
|
| 108 |
+
blockers: string[];
|
| 109 |
+
}>;
|
| 110 |
+
recommendations: Array<{ candidate: string; recommendation: string; blocker_count: number }>;
|
| 111 |
+
};
|
| 112 |
+
|
| 113 |
+
export const getContainerQuality = (runId: string, limit = 25) =>
|
| 114 |
+
get<ContainerQualityProfile>(
|
| 115 |
+
`/api/containers/quality?run_id=${encodeURIComponent(runId)}&limit=${limit}`
|
| 116 |
+
);
|
| 117 |
+
|
| 118 |
+
export const compareModels = (runIds: string[], baselineRunId?: string | null) =>
|
| 119 |
+
post<ModelCompareResponse>('/api/models/compare', {
|
| 120 |
+
run_ids: runIds,
|
| 121 |
+
baseline_run_id: baselineRunId ?? undefined,
|
| 122 |
+
});
|
| 123 |
+
|
| 124 |
+
export const getGateStatus = (params?: {
|
| 125 |
+
sequence_id?: string;
|
| 126 |
+
candidate_run_id?: string;
|
| 127 |
+
baseline_run_id?: string;
|
| 128 |
+
}) => {
|
| 129 |
+
const q = new URLSearchParams();
|
| 130 |
+
if (params?.sequence_id) q.set('sequence_id', params.sequence_id);
|
| 131 |
+
if (params?.candidate_run_id) q.set('candidate_run_id', params.candidate_run_id);
|
| 132 |
+
if (params?.baseline_run_id) q.set('baseline_run_id', params.baseline_run_id);
|
| 133 |
+
const qs = q.toString();
|
| 134 |
+
return get<GateStatusResponse>(`/api/gates/status${qs ? `?${qs}` : ''}`);
|
| 135 |
+
};
|
| 136 |
+
|
| 137 |
+
export const getEvaluationLevels = () =>
|
| 138 |
+
get<{ levels: string[]; backends: Record<string, string> }>('/api/evaluations/levels');
|
| 139 |
+
|
| 140 |
+
export const exportEvidence = (body: Record<string, unknown>) =>
|
| 141 |
+
post<{ package: Record<string, unknown>; path: string | null }>('/api/evaluations/evidence', body);
|
| 142 |
+
|
| 143 |
+
export function useContainerQuality(runId: string | null, refreshKey = 0) {
|
| 144 |
+
return usePoll(
|
| 145 |
+
() => (runId ? getContainerQuality(runId) : Promise.resolve(null)),
|
| 146 |
+
null,
|
| 147 |
+
[runId, refreshKey]
|
| 148 |
+
);
|
| 149 |
+
}
|
| 150 |
+
|
| 151 |
+
export function useGateStatus(
|
| 152 |
+
candidateRunId: string | null,
|
| 153 |
+
baselineRunId: string | null,
|
| 154 |
+
refreshKey = 0
|
| 155 |
+
) {
|
| 156 |
+
return usePoll(
|
| 157 |
+
() =>
|
| 158 |
+
getGateStatus({
|
| 159 |
+
candidate_run_id: candidateRunId ?? undefined,
|
| 160 |
+
baseline_run_id: baselineRunId ?? undefined,
|
| 161 |
+
}),
|
| 162 |
+
8000,
|
| 163 |
+
[candidateRunId, baselineRunId, refreshKey]
|
| 164 |
+
);
|
| 165 |
+
}
|
tests/test_platform/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Phase 1 platform foundation tests
|
tests/test_platform/test_foundation.py
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Phase 1 platform foundation tests — metrics, container quality, compare, gates, evidence."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import pytest
|
| 9 |
+
|
| 10 |
+
from sensorflow.platform.metrics_engine import (
|
| 11 |
+
aggregate_container_rows,
|
| 12 |
+
container_quality_metrics,
|
| 13 |
+
delta_metrics,
|
| 14 |
+
precision_recall_f1,
|
| 15 |
+
verification_rates,
|
| 16 |
+
)
|
| 17 |
+
from sensorflow.platform.container_quality import profile_from_counts
|
| 18 |
+
from sensorflow.platform.levels import AggregateLevel, EvaluationScope, summarize_scope
|
| 19 |
+
from sensorflow.platform.gates import (
|
| 20 |
+
DEFAULT_GATE_CONFIG,
|
| 21 |
+
evaluate_multi_gates,
|
| 22 |
+
list_gate_defs,
|
| 23 |
+
load_gate_config,
|
| 24 |
+
)
|
| 25 |
+
from sensorflow.platform.evidence import build_evidence_package, export_evidence_package
|
| 26 |
+
from sensorflow.platform.compare import compare_models
|
| 27 |
+
from sensorflow.platform.entities import EvidencePackage, Provenance
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
# ------------------------------------------------------------------ metrics
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def test_precision_recall_f1_basic():
|
| 34 |
+
out = precision_recall_f1(tp=8, fp=2, fn=2)
|
| 35 |
+
assert out["precision"] == 0.8
|
| 36 |
+
assert out["recall"] == 0.8
|
| 37 |
+
assert out["f1"] == 0.8
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def test_precision_recall_empty():
|
| 41 |
+
out = precision_recall_f1(0, 0, 0)
|
| 42 |
+
assert out["precision"] is None
|
| 43 |
+
assert out["recall"] is None
|
| 44 |
+
assert out["f1"] is None
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def test_verification_rates():
|
| 48 |
+
v = verification_rates(
|
| 49 |
+
n_objects=100,
|
| 50 |
+
verified=40,
|
| 51 |
+
reviewed=55,
|
| 52 |
+
disputed=5,
|
| 53 |
+
auto_accepted=30,
|
| 54 |
+
hitl=15,
|
| 55 |
+
)
|
| 56 |
+
assert v["verified_rate"] == 0.4
|
| 57 |
+
assert v["hitl_rate"] == 0.15
|
| 58 |
+
assert v["disputed_rate"] == 0.05
|
| 59 |
+
assert v["auto_accept_rate"] == 0.3
|
| 60 |
+
assert v["unverified"] == 60
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def test_container_quality_metrics_bundle():
|
| 64 |
+
m = container_quality_metrics(
|
| 65 |
+
tp=10, fp=5, fn=5, sum_iou=7.0, n_objects=20,
|
| 66 |
+
verified=8, reviewed=12, hitl=4, anomalies=2,
|
| 67 |
+
)
|
| 68 |
+
assert m["precision"] == pytest.approx(10 / 15, rel=1e-3)
|
| 69 |
+
assert m["mean_iou"] == 0.7
|
| 70 |
+
assert m["verification"]["verified_rate"] == 0.4
|
| 71 |
+
assert m["anomaly_rate"] == 0.1
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def test_aggregate_container_rows():
|
| 75 |
+
rows = [
|
| 76 |
+
{"tp": 5, "fp": 1, "fn": 1, "sum_iou": 4.0, "n_objects": 10, "verified": 3, "reviewed": 4},
|
| 77 |
+
{"tp": 5, "fp": 1, "fn": 1, "sum_iou": 3.5, "n_objects": 10, "verified": 5, "reviewed": 6},
|
| 78 |
+
]
|
| 79 |
+
agg = aggregate_container_rows(rows)
|
| 80 |
+
assert agg["tp"] == 10
|
| 81 |
+
assert agg["verification"]["verified"] == 8
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def test_delta_metrics():
|
| 85 |
+
deltas = delta_metrics(
|
| 86 |
+
{"precision": 0.9, "recall": 0.8},
|
| 87 |
+
{"precision": 0.88, "recall": 0.85},
|
| 88 |
+
keys=["precision", "recall"],
|
| 89 |
+
)
|
| 90 |
+
assert deltas[0]["delta"] == pytest.approx(-0.02)
|
| 91 |
+
assert deltas[1]["delta"] == pytest.approx(0.05)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def test_profile_from_counts():
|
| 95 |
+
p = profile_from_counts(
|
| 96 |
+
container_id="c-1", tp=4, fp=1, fn=1, sum_iou=3.2,
|
| 97 |
+
n_objects=10, verified=2, reviewed=5, hitl=3,
|
| 98 |
+
)
|
| 99 |
+
assert p["container_id"] == "c-1"
|
| 100 |
+
assert p["f1"] is not None
|
| 101 |
+
assert p["verification"]["hitl"] == 3
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
# ------------------------------------------------------------------ levels
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def test_aggregate_ladder():
|
| 108 |
+
ladder = AggregateLevel.ladder()
|
| 109 |
+
assert ladder[0] == AggregateLevel.FRAME
|
| 110 |
+
assert ladder[-1] == AggregateLevel.POPULATION
|
| 111 |
+
assert AggregateLevel.CONTAINER.parent() == AggregateLevel.DATASET
|
| 112 |
+
assert AggregateLevel.FRAME.child() is None
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def test_evaluation_scope_summary():
|
| 116 |
+
scope = EvaluationScope(
|
| 117 |
+
level=AggregateLevel.CONTAINER,
|
| 118 |
+
run_id="run-1",
|
| 119 |
+
mega_container_id=42,
|
| 120 |
+
model_version="model-v1",
|
| 121 |
+
)
|
| 122 |
+
s = summarize_scope(scope)
|
| 123 |
+
assert s["backend"] == "megaeval"
|
| 124 |
+
assert s["refs"]["mega_container_id"] == 42
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
# ------------------------------------------------------------------ gates
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def test_gate_defs_include_multi_gate_skeleton(tmp_path: Path):
|
| 131 |
+
cfg_path = tmp_path / "gates.json"
|
| 132 |
+
cfg = load_gate_config(cfg_path)
|
| 133 |
+
assert cfg_path.exists()
|
| 134 |
+
defs = list_gate_defs(cfg)
|
| 135 |
+
types = {d.gate_type for d in defs}
|
| 136 |
+
for required in ("scenario", "coverage", "regression", "safety", "release", "quality", "launch"):
|
| 137 |
+
assert required in types
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def test_evaluate_multi_gates_skeleton(tmp_path: Path, monkeypatch):
|
| 141 |
+
cfg_path = tmp_path / "gates.json"
|
| 142 |
+
monkeypatch.setattr(
|
| 143 |
+
"sensorflow.platform.gates.DEFAULT_GATE_CONFIG_PATH", cfg_path
|
| 144 |
+
)
|
| 145 |
+
# rewrite load to use tmp
|
| 146 |
+
from sensorflow.platform import gates as gates_mod
|
| 147 |
+
monkeypatch.setattr(gates_mod, "DEFAULT_GATE_CONFIG_PATH", cfg_path)
|
| 148 |
+
out = gates_mod.evaluate_multi_gates()
|
| 149 |
+
assert out["summary"]["unwired_count"] >= 3
|
| 150 |
+
assert out["summary"]["release_ready"] is False
|
| 151 |
+
types = [g["gate_type"] for g in out["gates"]]
|
| 152 |
+
assert "scenario" in types and "safety" in types
|
| 153 |
+
skeleton = [g for g in out["gates"] if g["gate_type"] == "safety"][0]
|
| 154 |
+
assert skeleton["ready"] is False
|
| 155 |
+
assert skeleton["passed"] is None
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def test_gate_config_thresholds_are_config_driven():
|
| 159 |
+
assert "map_3d" in DEFAULT_GATE_CONFIG["gates"]["quality"]["thresholds"]
|
| 160 |
+
assert DEFAULT_GATE_CONFIG["gates"]["safety"]["ready"] is False
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
# ------------------------------------------------------------------ evidence
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def test_evidence_package_stub(tmp_path: Path, monkeypatch):
|
| 167 |
+
monkeypatch.setattr(
|
| 168 |
+
"sensorflow.platform.evidence.EVIDENCE_DIR", tmp_path
|
| 169 |
+
)
|
| 170 |
+
pkg = build_evidence_package(evaluation_id="eval-test")
|
| 171 |
+
assert isinstance(pkg, EvidencePackage)
|
| 172 |
+
assert pkg.evaluation_id == "eval-test"
|
| 173 |
+
assert "model_versions" in pkg.placeholders or pkg.model_versions
|
| 174 |
+
path = export_evidence_package(pkg, tmp_path / f"{pkg.package_id}.json")
|
| 175 |
+
assert path.exists()
|
| 176 |
+
raw = json.loads(path.read_text())
|
| 177 |
+
assert raw["evaluation_id"] == "eval-test"
|
| 178 |
+
assert "gate_results" in raw
|
| 179 |
+
assert "compute_usage" in raw
|
| 180 |
+
assert "provenance" in raw
|
| 181 |
+
|
| 182 |
+
# append-only: second export with same path gets a revision file
|
| 183 |
+
path2 = export_evidence_package(pkg, path)
|
| 184 |
+
assert path2 != path
|
| 185 |
+
assert path2.exists()
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
# ------------------------------------------------------------------ compare (unit, no mega store)
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def test_compare_models_requires_two_runs():
|
| 192 |
+
with pytest.raises(ValueError):
|
| 193 |
+
compare_models(["only-one"])
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
class _FakeRun:
|
| 197 |
+
def __init__(self, run_id, model_version, headline, status="published"):
|
| 198 |
+
self.run_id = run_id
|
| 199 |
+
self.model_version = model_version
|
| 200 |
+
self.headline = headline
|
| 201 |
+
self.status = status
|
| 202 |
+
self.population_id = "pop-1"
|
| 203 |
+
self.label_version = "labels-v1"
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
class _FakeStore:
|
| 207 |
+
def __init__(self, runs):
|
| 208 |
+
self.runs = {r.run_id: r for r in runs}
|
| 209 |
+
|
| 210 |
+
def artifacts(self, run_id):
|
| 211 |
+
raise NotImplementedError
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def test_compare_models_matrix(monkeypatch):
|
| 215 |
+
a = _FakeRun("run-a", "model-A", {
|
| 216 |
+
"precision": 0.9, "recall": 0.8, "f1": 0.85,
|
| 217 |
+
"mean_iou": 0.7, "safety_recall": 0.75, "anomaly_rate": 0.1,
|
| 218 |
+
})
|
| 219 |
+
b = _FakeRun("run-b", "model-B", {
|
| 220 |
+
"precision": 0.88, "recall": 0.82, "f1": 0.85,
|
| 221 |
+
"mean_iou": 0.71, "safety_recall": 0.74, "anomaly_rate": 0.09,
|
| 222 |
+
})
|
| 223 |
+
c = _FakeRun("run-c", "model-C", {
|
| 224 |
+
"precision": 0.91, "recall": 0.79, "f1": 0.845,
|
| 225 |
+
"mean_iou": 0.69, "safety_recall": 0.78, "anomaly_rate": 0.11,
|
| 226 |
+
})
|
| 227 |
+
store = _FakeStore([a, b, c])
|
| 228 |
+
|
| 229 |
+
def fake_compare(store, cand, base, policy=None):
|
| 230 |
+
return {
|
| 231 |
+
"recommendation": "PROMOTE",
|
| 232 |
+
"blockers": [],
|
| 233 |
+
"headline_deltas": [],
|
| 234 |
+
"per_class": [],
|
| 235 |
+
"regressions": [],
|
| 236 |
+
}
|
| 237 |
+
|
| 238 |
+
monkeypatch.setattr("sensorflow.megaeval.analysis.compare_runs", fake_compare)
|
| 239 |
+
out = compare_models(
|
| 240 |
+
["run-a", "run-b", "run-c"],
|
| 241 |
+
baseline_run_id="run-a",
|
| 242 |
+
store=store,
|
| 243 |
+
)
|
| 244 |
+
assert len(out["models"]) == 3
|
| 245 |
+
assert len(out["pairwise"]) == 2
|
| 246 |
+
assert len(out["metric_matrix"]) >= 3
|
| 247 |
+
recall_row = next(r for r in out["metric_matrix"] if r["metric"] == "recall")
|
| 248 |
+
assert recall_row["model-B"] == 0.82
|
| 249 |
+
assert recall_row["deltas_vs_baseline"]["model-B"] == pytest.approx(0.02)
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
# ------------------------------------------------------------------ API smoke (optional FastAPI)
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def test_platform_router_mounts():
|
| 256 |
+
from fastapi.testclient import TestClient
|
| 257 |
+
from sensorflow.platform.api import router
|
| 258 |
+
from fastapi import FastAPI
|
| 259 |
+
|
| 260 |
+
app = FastAPI()
|
| 261 |
+
app.include_router(router)
|
| 262 |
+
client = TestClient(app)
|
| 263 |
+
levels = client.get("/api/evaluations/levels")
|
| 264 |
+
assert levels.status_code == 200
|
| 265 |
+
body = levels.json()
|
| 266 |
+
assert "population" in body["levels"]
|
| 267 |
+
assert body["backends"]["container"] == "megaeval"
|
| 268 |
+
|
| 269 |
+
gates = client.get("/api/gates")
|
| 270 |
+
assert gates.status_code == 200
|
| 271 |
+
assert any(g["gate_type"] == "scenario" for g in gates.json()["gates"])
|
| 272 |
+
|
| 273 |
+
status = client.get("/api/gates/status")
|
| 274 |
+
assert status.status_code == 200
|
| 275 |
+
assert status.json()["summary"]["release_ready"] is False
|
| 276 |
+
|
| 277 |
+
evidence = client.post("/api/evaluations/evidence", json={
|
| 278 |
+
"evaluation_id": "api-eval-1",
|
| 279 |
+
"persist": False,
|
| 280 |
+
})
|
| 281 |
+
assert evidence.status_code == 200
|
| 282 |
+
assert evidence.json()["package"]["evaluation_id"] == "api-eval-1"
|
| 283 |
+
|
| 284 |
+
bad = client.post("/api/models/compare", json={"run_ids": ["x"]})
|
| 285 |
+
assert bad.status_code == 422 or bad.status_code == 400
|