whassan commited on
Commit
f271df0
·
verified ·
1 Parent(s): a2bbd94

Deploy Sensorflow Studio

Browse files
.gitignore CHANGED
@@ -65,6 +65,7 @@ temp/
65
  # Generated runtime/test artifacts
66
  runs/labeleval/
67
  runs/megaeval/
 
68
  runs/pipeline/frame_test/
69
  runs/pipeline/qg_test/
70
  runs/pipeline/roundtrip/
 
65
  # Generated runtime/test artifacts
66
  runs/labeleval/
67
  runs/megaeval/
68
+ runs/platform/
69
  runs/pipeline/frame_test/
70
  runs/pipeline/qg_test/
71
  runs/pipeline/roundtrip/
app_backend.py CHANGED
@@ -1467,6 +1467,10 @@ app.include_router(labeleval_router)
1467
  from sensorflow.megaeval.api import router as megaeval_router
1468
  app.include_router(megaeval_router)
1469
 
 
 
 
 
1470
  # Mount static folder
1471
  static_dir = Path(__file__).parent / "static"
1472
  if static_dir.exists():
 
1467
  from sensorflow.megaeval.api import router as megaeval_router
1468
  app.include_router(megaeval_router)
1469
 
1470
+ # Population-scale foundation (container quality, multi-model compare, multi-gate, evidence).
1471
+ from sensorflow.platform.api import router as platform_router
1472
+ app.include_router(platform_router)
1473
+
1474
  # Mount static folder
1475
  static_dir = Path(__file__).parent / "static"
1476
  if static_dir.exists():
sensorflow/platform/__init__.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Population-scale evaluation foundation — extends MegaEval + LabelEval.
2
+
3
+ Phase 1: schemas, aggregate levels, metrics, container quality, model compare,
4
+ multi-gate skeleton, evidence packages. Phases 2–7 leave TODO hooks only.
5
+ """
6
+
7
+ from sensorflow.platform.levels import AggregateLevel, EvaluationScope
8
+ from sensorflow.platform.entities import Provenance, EvidencePackage
9
+
10
+ __all__ = [
11
+ "AggregateLevel",
12
+ "EvaluationScope",
13
+ "Provenance",
14
+ "EvidencePackage",
15
+ ]
sensorflow/platform/api.py ADDED
@@ -0,0 +1,237 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Platform Phase 1 FastAPI routes — mount into existing app_backend.
2
+
3
+ Routes:
4
+ /api/evaluations/...
5
+ /api/containers/...
6
+ /api/models/compare
7
+ /api/gates/...
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from typing import Any, Dict, List, Optional
13
+
14
+ from fastapi import APIRouter, HTTPException
15
+ from pydantic import BaseModel, Field
16
+
17
+ from sensorflow.platform import levels
18
+ from sensorflow.platform.container_quality import build_container_quality_profile
19
+ from sensorflow.platform.compare import compare_models
20
+ from sensorflow.platform.evidence import build_evidence_package, export_evidence_package
21
+ from sensorflow.platform.gates import evaluate_multi_gates, list_gate_defs, load_gate_config, save_gate_config
22
+ from sensorflow.platform.levels import AggregateLevel, EvaluationScope
23
+
24
+ router = APIRouter(prefix="/api")
25
+
26
+
27
+ # ------------------------------------------------------------------ requests
28
+
29
+
30
+ class EvaluationCreateRequest(BaseModel):
31
+ level: AggregateLevel = AggregateLevel.POPULATION
32
+ dataset_id: Optional[str] = None
33
+ population_id: Optional[str] = None
34
+ run_id: Optional[str] = None
35
+ container_id: Optional[str] = None
36
+ mega_container_id: Optional[int] = None
37
+ drive_id: Optional[str] = None
38
+ scene_id: Optional[str] = None
39
+ sequence_id: Optional[str] = None
40
+ frame_id: Optional[str] = None
41
+ cohort_filters: Dict[str, List[str]] = Field(default_factory=dict)
42
+ model_version: Optional[str] = None
43
+ label_version: Optional[str] = None
44
+
45
+
46
+ class ModelCompareRequest(BaseModel):
47
+ run_ids: List[str] = Field(..., min_length=2)
48
+ baseline_run_id: Optional[str] = None
49
+ policy: Dict[str, Any] = Field(default_factory=dict)
50
+
51
+
52
+ class GateEvaluateRequest(BaseModel):
53
+ sequence_id: Optional[str] = None
54
+ candidate_run_id: Optional[str] = None
55
+ baseline_run_id: Optional[str] = None
56
+
57
+
58
+ class GateConfigUpdate(BaseModel):
59
+ gates: Dict[str, Any] = Field(default_factory=dict)
60
+
61
+
62
+ class EvidenceExportRequest(BaseModel):
63
+ evaluation_id: Optional[str] = None
64
+ run_id: Optional[str] = None
65
+ dataset_id: Optional[str] = None
66
+ sequence_id: Optional[str] = None
67
+ candidate_run_id: Optional[str] = None
68
+ baseline_run_id: Optional[str] = None
69
+ persist: bool = True
70
+
71
+
72
+ # ------------------------------------------------------------------ evaluations
73
+
74
+
75
+ @router.get("/evaluations/levels")
76
+ def evaluation_levels():
77
+ return {
78
+ "levels": [lvl.value for lvl in AggregateLevel.ladder()],
79
+ "backends": {lvl.value: levels.LEVEL_BACKEND[lvl] for lvl in AggregateLevel},
80
+ }
81
+
82
+
83
+ @router.post("/evaluations/scope")
84
+ def create_evaluation_scope(req: EvaluationCreateRequest):
85
+ scope = EvaluationScope(**req.model_dump())
86
+ return {
87
+ "scope": levels.summarize_scope(scope),
88
+ "message": "Phase 1 scope descriptor — queries still served by MegaEval/LabelEval backends",
89
+ }
90
+
91
+
92
+ @router.get("/evaluations/{run_id}")
93
+ def get_evaluation_summary(run_id: str):
94
+ """Unified evaluation summary for a MegaEval run (async job status preserved)."""
95
+ try:
96
+ from sensorflow.megaeval.runs import get_mega_store
97
+ run = get_mega_store().runs.get(run_id)
98
+ except Exception as exc:
99
+ raise HTTPException(500, str(exc)) from exc
100
+ if run is None:
101
+ raise HTTPException(404, f"Unknown evaluation run {run_id}")
102
+ scope = EvaluationScope(
103
+ level=AggregateLevel.POPULATION,
104
+ evaluation_id=run.run_id,
105
+ run_id=run.run_id,
106
+ population_id=run.population_id,
107
+ model_version=run.model_version,
108
+ label_version=getattr(run, "label_version", None),
109
+ )
110
+ return {
111
+ "evaluation_id": run.run_id,
112
+ "status": run.status,
113
+ "scope": levels.summarize_scope(scope),
114
+ "headline": run.headline,
115
+ "model_version": run.model_version,
116
+ "population_id": run.population_id,
117
+ "lineage": getattr(run, "lineage", None),
118
+ "progress": getattr(run, "progress", None),
119
+ }
120
+
121
+
122
+ @router.post("/evaluations/evidence")
123
+ def export_evidence(req: EvidenceExportRequest):
124
+ package = build_evidence_package(
125
+ evaluation_id=req.evaluation_id,
126
+ run_id=req.run_id,
127
+ dataset_id=req.dataset_id,
128
+ sequence_id=req.sequence_id,
129
+ candidate_run_id=req.candidate_run_id,
130
+ baseline_run_id=req.baseline_run_id,
131
+ )
132
+ path = None
133
+ if req.persist:
134
+ path = str(export_evidence_package(package))
135
+ return {"package": package.model_dump(), "path": path}
136
+
137
+
138
+ # ------------------------------------------------------------------ containers
139
+
140
+
141
+ @router.get("/containers/quality")
142
+ def containers_quality(
143
+ run_id: str,
144
+ sort: str = "highest_risk",
145
+ limit: int = 50,
146
+ offset: int = 0,
147
+ ):
148
+ try:
149
+ return build_container_quality_profile(
150
+ run_id, sort=sort, limit=limit, offset=offset
151
+ )
152
+ except KeyError as exc:
153
+ raise HTTPException(404, str(exc)) from exc
154
+ except RuntimeError as exc:
155
+ raise HTTPException(409, str(exc)) from exc
156
+
157
+
158
+ @router.get("/containers/{container_id}/quality")
159
+ def container_quality(container_id: int, run_id: str):
160
+ try:
161
+ return build_container_quality_profile(run_id, container_id=container_id)
162
+ except KeyError as exc:
163
+ raise HTTPException(404, str(exc)) from exc
164
+ except RuntimeError as exc:
165
+ raise HTTPException(409, str(exc)) from exc
166
+
167
+
168
+ # ------------------------------------------------------------------ models compare
169
+
170
+
171
+ @router.post("/models/compare")
172
+ def models_compare(req: ModelCompareRequest):
173
+ try:
174
+ return compare_models(
175
+ req.run_ids,
176
+ baseline_run_id=req.baseline_run_id,
177
+ policy=req.policy or None,
178
+ )
179
+ except ValueError as exc:
180
+ raise HTTPException(400, str(exc)) from exc
181
+ except KeyError as exc:
182
+ raise HTTPException(404, str(exc)) from exc
183
+ except RuntimeError as exc:
184
+ raise HTTPException(409, str(exc)) from exc
185
+
186
+
187
+ # ------------------------------------------------------------------ gates
188
+
189
+
190
+ @router.get("/gates")
191
+ def gates_list():
192
+ cfg = load_gate_config()
193
+ return {
194
+ "config_path": str(cfg.get("schema_version")),
195
+ "gates": [g.model_dump() for g in list_gate_defs(cfg)],
196
+ "order": ["scenario", "coverage", "regression", "safety", "quality", "launch", "release"],
197
+ }
198
+
199
+
200
+ @router.get("/gates/config")
201
+ def gates_config():
202
+ return load_gate_config()
203
+
204
+
205
+ @router.post("/gates/config")
206
+ def gates_config_update(req: GateConfigUpdate):
207
+ cfg = load_gate_config()
208
+ for name, patch in req.gates.items():
209
+ if name not in cfg["gates"]:
210
+ cfg["gates"][name] = patch
211
+ else:
212
+ cfg["gates"][name].update(patch)
213
+ save_gate_config(cfg)
214
+ return cfg
215
+
216
+
217
+ @router.post("/gates/evaluate")
218
+ def gates_evaluate(req: GateEvaluateRequest):
219
+ return evaluate_multi_gates(
220
+ sequence_id=req.sequence_id,
221
+ candidate_run_id=req.candidate_run_id,
222
+ baseline_run_id=req.baseline_run_id,
223
+ )
224
+
225
+
226
+ @router.get("/gates/status")
227
+ def gates_status(
228
+ sequence_id: Optional[str] = None,
229
+ candidate_run_id: Optional[str] = None,
230
+ baseline_run_id: Optional[str] = None,
231
+ ):
232
+ """Multi-gate readiness skeleton for UI cards."""
233
+ return evaluate_multi_gates(
234
+ sequence_id=sequence_id,
235
+ candidate_run_id=candidate_run_id,
236
+ baseline_run_id=baseline_run_id,
237
+ )
sensorflow/platform/compare.py ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Multi-model comparison (A vs B vs C) wrapping MegaEval comparative analytics.
2
+
3
+ Does not invent a second compare engine — delegates pairwise compare_runs and
4
+ folds results into a multi-way delta table.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Any, Dict, List, Optional
10
+
11
+ from sensorflow.megaeval import analysis
12
+ from sensorflow.megaeval.runs import MegaStore, get_mega_store
13
+ from sensorflow.platform.metrics_engine import delta_metrics, round4
14
+
15
+
16
+ def _headline_map(run) -> Dict[str, Optional[float]]:
17
+ h = run.headline or {}
18
+ return {k: h.get(k) for k in (
19
+ "precision", "recall", "f1", "mean_iou", "safety_recall", "anomaly_rate"
20
+ )}
21
+
22
+
23
+ def compare_models(
24
+ run_ids: List[str],
25
+ *,
26
+ baseline_run_id: Optional[str] = None,
27
+ policy: Optional[Dict] = None,
28
+ store: Optional[MegaStore] = None,
29
+ ) -> Dict[str, Any]:
30
+ """Compare 2+ published MegaEval runs (model versions A/B/C…).
31
+
32
+ If baseline_run_id is omitted, the first id is the baseline.
33
+ """
34
+ if len(run_ids) < 2:
35
+ raise ValueError("compare_models requires at least two run_ids")
36
+ store = store or get_mega_store()
37
+ baseline_id = baseline_run_id or run_ids[0]
38
+ if baseline_id not in run_ids:
39
+ run_ids = [baseline_id, *run_ids]
40
+
41
+ runs = []
42
+ for rid in run_ids:
43
+ run = store.runs.get(rid)
44
+ if run is None:
45
+ raise KeyError(f"Unknown evaluation run {rid}")
46
+ if run.status != "published":
47
+ raise RuntimeError(f"Run {rid} is {run.status}; results require published")
48
+ runs.append(run)
49
+
50
+ baseline = next(r for r in runs if r.run_id == baseline_id)
51
+ candidates = [r for r in runs if r.run_id != baseline_id]
52
+
53
+ models = [{
54
+ "run_id": r.run_id,
55
+ "model_version": r.model_version,
56
+ "role": "baseline" if r.run_id == baseline_id else "candidate",
57
+ "headline": _headline_map(r),
58
+ } for r in runs]
59
+
60
+ # Multi-way metric table: one row per metric, columns per model
61
+ metric_keys = ["precision", "recall", "f1", "mean_iou", "safety_recall", "anomaly_rate"]
62
+ matrix = []
63
+ for key in metric_keys:
64
+ row: Dict[str, Any] = {"metric": key}
65
+ for r in runs:
66
+ row[r.model_version] = round4(r.headline.get(key)) if r.headline.get(key) is not None else None
67
+ base_v = baseline.headline.get(key)
68
+ deltas = {}
69
+ for c in candidates:
70
+ cv = c.headline.get(key)
71
+ if base_v is not None and cv is not None:
72
+ deltas[c.model_version] = round4(float(cv) - float(base_v))
73
+ row["deltas_vs_baseline"] = deltas
74
+ matrix.append(row)
75
+
76
+ pairwise = []
77
+ recommendations = []
78
+ for cand in candidates:
79
+ cmp = analysis.compare_runs(store, cand, baseline, policy)
80
+ pairwise.append({
81
+ "baseline_run_id": baseline.run_id,
82
+ "candidate_run_id": cand.run_id,
83
+ "baseline_model": baseline.model_version,
84
+ "candidate_model": cand.model_version,
85
+ "recommendation": cmp.get("recommendation"),
86
+ "blockers": cmp.get("blockers") or [],
87
+ "headline": cmp.get("headline_deltas") or delta_metrics(
88
+ _headline_map(baseline), _headline_map(cand)
89
+ ),
90
+ "per_class": cmp.get("per_class") or [],
91
+ "regressions": (cmp.get("regressions") or cmp.get("worst_cohorts") or [])[:10],
92
+ })
93
+ recommendations.append({
94
+ "candidate": cand.model_version,
95
+ "recommendation": cmp.get("recommendation"),
96
+ "blocker_count": len(cmp.get("blockers") or []),
97
+ })
98
+
99
+ # Also surface LabelEval registry models when present (non-fatal)
100
+ registry = []
101
+ try:
102
+ from sensorflow.evaluation.records import get_store
103
+ for m in get_store().all("models"):
104
+ registry.append({
105
+ "model_id": m.model_id,
106
+ "model_version": m.model_version,
107
+ "metrics": m.metrics.model_dump() if hasattr(m.metrics, "model_dump") else {},
108
+ "regression_status": m.regression_status,
109
+ })
110
+ except Exception:
111
+ pass
112
+
113
+ return {
114
+ "baseline_run_id": baseline.run_id,
115
+ "run_ids": run_ids,
116
+ "models": models,
117
+ "metric_matrix": matrix,
118
+ "pairwise": pairwise,
119
+ "recommendations": recommendations,
120
+ "label_eval_registry": registry,
121
+ "policy": {**analysis.DEFAULT_PROMOTION_POLICY, **(policy or {})},
122
+ }
sensorflow/platform/container_quality.py ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Container quality profiles — wraps MegaEval container table + metric engine."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any, Dict, List, Optional
6
+
7
+ import numpy as np
8
+
9
+ from sensorflow.megaeval import population as pop_mod
10
+ from sensorflow.megaeval.population import DIMENSIONS
11
+ from sensorflow.megaeval.runs import CONTAINER_STATUS, EvaluationRun, MegaStore, get_mega_store
12
+ from sensorflow.platform.metrics_engine import container_quality_metrics, aggregate_container_rows
13
+
14
+
15
+ def _enrich_row(r: Dict[str, Any]) -> Dict[str, Any]:
16
+ tp, fp, fn = int(r["tp"]), int(r["fp"]), int(r["fn"])
17
+ sum_iou = float(r.get("sum_iou") or 0.0)
18
+ n = int(r.get("n_objects") or 0)
19
+ verified = int(r.get("verified") or 0)
20
+ reviewed = int(r.get("reviewed") or 0)
21
+ # MegaEval does not yet track disputed/auto_accept/hitl separately;
22
+ # approximate: reviewed-verified ≈ HITL queue residue; rest unverified.
23
+ hitl = max(0, reviewed - verified)
24
+ auto_accepted = 0 # TODO(Phase 2): wire triage auto-accept
25
+ disputed = 0 # TODO(Phase 2): grader disagreement → disputed
26
+ metrics = container_quality_metrics(
27
+ tp=tp, fp=fp, fn=fn, sum_iou=sum_iou, n_objects=n,
28
+ verified=verified, reviewed=reviewed, disputed=disputed,
29
+ auto_accepted=auto_accepted, hitl=hitl,
30
+ anomalies=int(r.get("anomalies") or 0),
31
+ )
32
+ dims = {dim: DIMENSIONS[dim][int(r[dim])] for dim in pop_mod.CONTAINER_DIMS if dim in r}
33
+ return {
34
+ "container_id": int(r["container_id"]),
35
+ "status": CONTAINER_STATUS[int(r["status"])] if "status" in r else None,
36
+ "risk_score": round(float(r.get("risk_score") or 0), 4),
37
+ "dims": dims,
38
+ "drill_down": {
39
+ "object_endpoint": f"/api/megaeval/runs/{{run_id}}/containers/{int(r['container_id'])}/objects",
40
+ "annotation_ids": [], # forensic load on demand
41
+ },
42
+ **metrics,
43
+ }
44
+
45
+
46
+ def build_container_quality_profile(
47
+ run_id: str,
48
+ *,
49
+ container_id: Optional[int] = None,
50
+ sort: str = "highest_risk",
51
+ limit: int = 50,
52
+ offset: int = 0,
53
+ store: Optional[MegaStore] = None,
54
+ ) -> Dict[str, Any]:
55
+ store = store or get_mega_store()
56
+ run = store.runs.get(run_id)
57
+ if run is None:
58
+ raise KeyError(f"Unknown evaluation run {run_id}")
59
+ if run.status != "published":
60
+ raise RuntimeError(f"Run {run_id} is {run.status}; publish before quality profile")
61
+
62
+ art = store.artifacts(run_id)
63
+ df = art["containers"].copy()
64
+ tp, fp, fn = df["tp"], df["fp"], df["fn"]
65
+ with np.errstate(divide="ignore", invalid="ignore"):
66
+ df["recall"] = np.where((tp + fn) > 0, tp / (tp + fn), np.nan)
67
+ df["precision"] = np.where((tp + fp) > 0, tp / (tp + fp), np.nan)
68
+ df["mean_iou"] = np.where(tp > 0, df["sum_iou"] / tp, np.nan)
69
+
70
+ if container_id is not None:
71
+ df = df[df["container_id"] == int(container_id)]
72
+ if df.empty:
73
+ raise KeyError(f"Unknown container {container_id} in run {run_id}")
74
+
75
+ sort_map = {
76
+ "highest_risk": ("risk_score", False),
77
+ "worst_recall": ("recall", True),
78
+ "worst_precision": ("precision", True),
79
+ "worst_iou": ("mean_iou", True),
80
+ "most_anomalies": ("anomalies", False),
81
+ "least_verified": ("verified", True),
82
+ }
83
+ metric, ascending = sort_map.get(sort, sort_map["highest_risk"])
84
+ df = df.sort_values(metric, ascending=ascending, na_position="last")
85
+ total = len(df)
86
+ page = df.iloc[offset: offset + max(1, min(limit, 500))]
87
+
88
+ rows: List[Dict[str, Any]] = []
89
+ for _, series in page.iterrows():
90
+ rows.append(_enrich_row(series.to_dict()))
91
+
92
+ summary = aggregate_container_rows(rows) if rows else container_quality_metrics(
93
+ tp=0, fp=0, fn=0
94
+ )
95
+
96
+ return {
97
+ "run_id": run_id,
98
+ "model_version": run.model_version,
99
+ "population_id": run.population_id,
100
+ "container_id": container_id,
101
+ "total": total,
102
+ "sort": sort,
103
+ "offset": offset,
104
+ "limit": limit,
105
+ "summary": summary,
106
+ "containers": rows,
107
+ "verification_headline": summary.get("verification"),
108
+ }
109
+
110
+
111
+ def profile_from_counts(
112
+ *,
113
+ container_id: str,
114
+ tp: int,
115
+ fp: int,
116
+ fn: int,
117
+ sum_iou: float = 0.0,
118
+ n_objects: int = 0,
119
+ verified: int = 0,
120
+ reviewed: int = 0,
121
+ disputed: int = 0,
122
+ auto_accepted: int = 0,
123
+ hitl: int = 0,
124
+ anomalies: int = 0,
125
+ ) -> Dict[str, Any]:
126
+ """Unit-testable path without MegaStore."""
127
+ metrics = container_quality_metrics(
128
+ tp=tp, fp=fp, fn=fn, sum_iou=sum_iou, n_objects=n_objects or (tp + fp),
129
+ verified=verified, reviewed=reviewed, disputed=disputed,
130
+ auto_accepted=auto_accepted, hitl=hitl, anomalies=anomalies,
131
+ )
132
+ return {"container_id": container_id, **metrics}
sensorflow/platform/entities.py ADDED
@@ -0,0 +1,306 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Platform entity schemas with versioning / provenance.
2
+
3
+ Prefer these as the shared vocabulary across LabelEval + MegaEval.
4
+ Existing store models in evaluation.records remain the persistence layer for
5
+ label QA; MegaEval continues to use npz cubes. These Pydantic models are the
6
+ contract for APIs, evidence packages, and later lakehouse rows.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from datetime import datetime, timezone
12
+ from typing import Any, Dict, List, Literal, Optional
13
+ from uuid import uuid4
14
+
15
+ from pydantic import BaseModel, Field
16
+
17
+
18
+ def _now() -> str:
19
+ return datetime.now(timezone.utc).isoformat()
20
+
21
+
22
+ def _id(prefix: str) -> str:
23
+ return f"{prefix}-{uuid4().hex[:10]}"
24
+
25
+
26
+ class Provenance(BaseModel):
27
+ """Lineage fields required on durable evaluation artifacts."""
28
+
29
+ created_at: str = Field(default_factory=_now)
30
+ created_by: str = "system"
31
+ schema_version: str = "platform-1.0"
32
+ parent_ids: List[str] = Field(default_factory=list)
33
+ source_system: str = "sensorflow" # labeleval | megaeval | pipeline | platform
34
+ seed: Optional[int] = None
35
+ code_version: str = "phase1"
36
+ notes: str = ""
37
+
38
+
39
+ class VersionedEntity(BaseModel):
40
+ version: str = "v1"
41
+ provenance: Provenance = Field(default_factory=Provenance)
42
+
43
+
44
+ # ------------------------------------------------------------------ hierarchy
45
+
46
+
47
+ class DatasetEntity(VersionedEntity):
48
+ dataset_id: str
49
+ name: str = ""
50
+ population_id: Optional[str] = None # MegaEval link
51
+ num_frames: int = 0
52
+ gt_type: Optional[str] = None
53
+
54
+
55
+ class Container(VersionedEntity):
56
+ """Scene/segment holding related objects (MegaEval container_id)."""
57
+
58
+ container_id: str
59
+ dataset_id: Optional[str] = None
60
+ run_id: Optional[str] = None
61
+ mega_container_id: Optional[int] = None
62
+ drive_id: Optional[str] = None
63
+ scene_id: Optional[str] = None
64
+ dims: Dict[str, str] = Field(default_factory=dict)
65
+ n_objects: int = 0
66
+
67
+
68
+ class Group(VersionedEntity):
69
+ """Logical grouping: verification bucket OR cohort alias."""
70
+
71
+ group_id: str
72
+ kind: Literal["verification", "cohort", "custom"] = "custom"
73
+ dataset_id: Optional[str] = None
74
+ member_ids: List[str] = Field(default_factory=list)
75
+ label: str = ""
76
+
77
+
78
+ class Drive(VersionedEntity):
79
+ drive_id: str
80
+ dataset_id: Optional[str] = None
81
+ scene_ids: List[str] = Field(default_factory=list)
82
+ container_ids: List[str] = Field(default_factory=list)
83
+ description: str = ""
84
+
85
+
86
+ class SceneEntity(VersionedEntity):
87
+ scene_id: str
88
+ dataset_id: Optional[str] = None
89
+ drive_id: Optional[str] = None
90
+ sequence_ids: List[str] = Field(default_factory=list)
91
+ description: str = ""
92
+
93
+
94
+ class ScenarioEntity(VersionedEntity):
95
+ scenario_id: str
96
+ scenario_type: str
97
+ dataset_id: Optional[str] = None
98
+ frame_ids: List[str] = Field(default_factory=list)
99
+ description: str = ""
100
+
101
+
102
+ class FrameEntity(VersionedEntity):
103
+ frame_id: str
104
+ dataset_id: Optional[str] = None
105
+ scene_id: Optional[str] = None
106
+ sequence_id: Optional[str] = None
107
+ container_id: Optional[str] = None
108
+ index: int = 0
109
+ timestamp_us: int = 0
110
+
111
+
112
+ class Sensor(VersionedEntity):
113
+ sensor_id: str
114
+ modality: Literal["lidar", "camera", "radar", "ultrasonic", "other"] = "lidar"
115
+ frame_id: Optional[str] = None
116
+ calibration_ref: Optional[str] = None
117
+
118
+
119
+ class ObjectEntity(VersionedEntity):
120
+ object_id: str
121
+ class_name: str = "vehicle"
122
+ frame_id: Optional[str] = None
123
+ container_id: Optional[str] = None
124
+ track_id: Optional[str] = None
125
+ bbox_3d: Optional[List[float]] = None
126
+
127
+
128
+ class Trajectory(VersionedEntity):
129
+ trajectory_id: str
130
+ track_id: Optional[str] = None
131
+ object_ids: List[str] = Field(default_factory=list)
132
+ frame_ids: List[str] = Field(default_factory=list)
133
+
134
+
135
+ class Label(VersionedEntity):
136
+ """Alias contract for Annotation — hypothesis, never assumed correct."""
137
+
138
+ label_id: str
139
+ annotation_id: Optional[str] = None
140
+ frame_id: Optional[str] = None
141
+ object_id: Optional[str] = None
142
+ class_name: str = ""
143
+ confidence: float = 0.0
144
+ status: str = "PENDING"
145
+ model_version: Optional[str] = None
146
+
147
+
148
+ class AnnotationEntity(Label):
149
+ """Explicit Annotation mirror for API clarity."""
150
+
151
+ pass
152
+
153
+
154
+ # ------------------------------------------------------------------ models / eval
155
+
156
+
157
+ class ModelEntity(VersionedEntity):
158
+ model_id: str
159
+ name: str = ""
160
+ status: str = "ready"
161
+
162
+
163
+ class ModelVersion(VersionedEntity):
164
+ model_version_id: str
165
+ model_id: str
166
+ model_version: str
167
+ checkpoint_ref: Optional[str] = None
168
+ trained_on_dataset: Optional[str] = None
169
+ metrics: Dict[str, Any] = Field(default_factory=dict)
170
+
171
+
172
+ class Metric(VersionedEntity):
173
+ metric_id: str
174
+ name: str
175
+ value: Optional[float] = None
176
+ unit: str = "ratio"
177
+ higher_is_better: bool = True
178
+ support: Optional[int] = None
179
+ level: str = "container" # AggregateLevel value
180
+ exact: bool = True
181
+
182
+
183
+ class Evaluation(VersionedEntity):
184
+ evaluation_id: str
185
+ level: str
186
+ dataset_id: Optional[str] = None
187
+ population_id: Optional[str] = None
188
+ run_id: Optional[str] = None
189
+ model_version: Optional[str] = None
190
+ label_version: Optional[str] = None
191
+ scope_ref: Optional[str] = None # container/cohort/drive id
192
+ metrics: List[Metric] = Field(default_factory=list)
193
+ status: str = "created"
194
+
195
+
196
+ class Cohort(VersionedEntity):
197
+ cohort_id: str
198
+ filters: Dict[str, List[str]] = Field(default_factory=dict)
199
+ group_by: List[str] = Field(default_factory=list)
200
+ n: int = 0
201
+ run_id: Optional[str] = None
202
+
203
+
204
+ class Embedding(VersionedEntity):
205
+ embedding_id: str
206
+ ref_type: str = "container" # container | object | frame
207
+ ref_id: str
208
+ dim: int = 0
209
+ # TODO(Phase 3): vector payload / external vector-DB handle
210
+ vector_ref: Optional[str] = None
211
+
212
+
213
+ class Anomaly(VersionedEntity):
214
+ anomaly_id: str
215
+ annotation_id: Optional[str] = None
216
+ container_id: Optional[str] = None
217
+ score: float = 0.0
218
+ is_anomaly: bool = False
219
+ detector_scores: Dict[str, float] = Field(default_factory=dict)
220
+
221
+
222
+ class Conflict(VersionedEntity):
223
+ """Safety / label conflict placeholder — Phase 4 fills SSAM DRAC/DeltaS."""
224
+
225
+ conflict_id: str
226
+ kind: str = "label" # label | trajectory | ssam_drac | ssam_deltas
227
+ entity_ids: List[str] = Field(default_factory=list)
228
+ severity: str = "medium"
229
+ # TODO(Phase 4): full SSAM conflict engine fields
230
+ detail: Dict[str, Any] = Field(default_factory=dict)
231
+
232
+
233
+ class ODDDefinition(VersionedEntity):
234
+ odd_id: str
235
+ name: str
236
+ axes: Dict[str, List[str]] = Field(default_factory=dict)
237
+ # TODO(Phase 5): combinatorial coverage rules
238
+ description: str = ""
239
+
240
+
241
+ class ODDObservation(VersionedEntity):
242
+ observation_id: str
243
+ odd_id: str
244
+ observed_axes: Dict[str, str] = Field(default_factory=dict)
245
+ covered: bool = False
246
+ support: int = 0
247
+ run_id: Optional[str] = None
248
+
249
+
250
+ class QualityGateDef(VersionedEntity):
251
+ gate_id: str
252
+ name: str
253
+ gate_type: Literal[
254
+ "scenario", "coverage", "regression", "safety", "release", "quality", "launch"
255
+ ]
256
+ thresholds: Dict[str, float] = Field(default_factory=dict)
257
+ config_path: Optional[str] = None
258
+ enabled: bool = True
259
+
260
+
261
+ class GateResult(VersionedEntity):
262
+ result_id: str
263
+ gate_id: str
264
+ gate_type: str
265
+ passed: Optional[bool] = None # None = not yet evaluated (skeleton)
266
+ failures: List[Dict[str, Any]] = Field(default_factory=list)
267
+ metrics: Dict[str, Any] = Field(default_factory=dict)
268
+ thresholds: Dict[str, float] = Field(default_factory=dict)
269
+ scope_ref: Optional[str] = None
270
+ ready: bool = False # False until Phase N wires real checks
271
+ message: str = ""
272
+
273
+
274
+ class HumanReviewEntity(VersionedEntity):
275
+ review_id: str
276
+ task_id: Optional[str] = None
277
+ annotation_id: Optional[str] = None
278
+ reviewer: str = "human-reviewer"
279
+ action: Dict[str, Any] = Field(default_factory=dict)
280
+
281
+
282
+ class ComputeUsage(VersionedEntity):
283
+ usage_id: str = Field(default_factory=lambda: _id("compute"))
284
+ stage: str = ""
285
+ units: int = 0
286
+ run_id: str = ""
287
+ evaluation_id: Optional[str] = None
288
+ hardware: str = "cpu"
289
+
290
+
291
+ class EvidencePackage(VersionedEntity):
292
+ """Exportable evaluation artifact — Phase 1 stub with real fields when available."""
293
+
294
+ package_id: str = Field(default_factory=lambda: _id("evidence"))
295
+ evaluation_id: str
296
+ dataset_id: Optional[str] = None
297
+ dataset_version: Optional[str] = None
298
+ population_id: Optional[str] = None
299
+ model_versions: List[str] = Field(default_factory=list)
300
+ label_version: Optional[str] = None
301
+ metrics: Dict[str, Any] = Field(default_factory=dict)
302
+ thresholds: Dict[str, Any] = Field(default_factory=dict)
303
+ gate_results: List[Dict[str, Any]] = Field(default_factory=list)
304
+ compute_usage: List[Dict[str, Any]] = Field(default_factory=list)
305
+ drill_down_ids: Dict[str, List[str]] = Field(default_factory=dict)
306
+ placeholders: List[str] = Field(default_factory=list)
sensorflow/platform/evidence.py ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Evidence package export — JSON stub with real fields when available."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+ from typing import Any, Dict, List, Optional
8
+
9
+ from sensorflow.platform.entities import EvidencePackage, Provenance, _id
10
+
11
+ EVIDENCE_DIR = Path("runs/platform/evidence")
12
+
13
+
14
+ def build_evidence_package(
15
+ *,
16
+ evaluation_id: Optional[str] = None,
17
+ run_id: Optional[str] = None,
18
+ dataset_id: Optional[str] = None,
19
+ sequence_id: Optional[str] = None,
20
+ candidate_run_id: Optional[str] = None,
21
+ baseline_run_id: Optional[str] = None,
22
+ include_gates: bool = True,
23
+ ) -> EvidencePackage:
24
+ placeholders: List[str] = []
25
+ metrics: Dict[str, Any] = {}
26
+ model_versions: List[str] = []
27
+ population_id = None
28
+ label_version = None
29
+ dataset_version = None
30
+ compute_usage: List[Dict[str, Any]] = []
31
+ drill: Dict[str, List[str]] = {}
32
+ thresholds: Dict[str, Any] = {}
33
+ gate_results: List[Dict[str, Any]] = []
34
+
35
+ eval_id = evaluation_id or run_id or _id("eval")
36
+
37
+ # MegaEval run headline
38
+ if run_id or candidate_run_id:
39
+ rid = run_id or candidate_run_id
40
+ try:
41
+ from sensorflow.megaeval.runs import get_mega_store
42
+ store = get_mega_store()
43
+ run = store.runs.get(rid) # type: ignore[arg-type]
44
+ if run:
45
+ metrics["headline"] = dict(run.headline or {})
46
+ model_versions.append(run.model_version)
47
+ population_id = run.population_id
48
+ label_version = getattr(run, "label_version", None)
49
+ drill["run_ids"] = [run.run_id]
50
+ else:
51
+ placeholders.append(f"mega_run_missing:{rid}")
52
+ except Exception as exc:
53
+ placeholders.append(f"mega_run_error:{exc}")
54
+ else:
55
+ placeholders.append("run_id")
56
+
57
+ if baseline_run_id:
58
+ try:
59
+ from sensorflow.megaeval.runs import get_mega_store
60
+ br = get_mega_store().runs.get(baseline_run_id)
61
+ if br:
62
+ model_versions.append(br.model_version)
63
+ drill.setdefault("run_ids", []).append(br.run_id)
64
+ except Exception:
65
+ placeholders.append("baseline_run")
66
+
67
+ # LabelEval dataset + process units
68
+ if dataset_id:
69
+ try:
70
+ from sensorflow.evaluation.records import get_store
71
+ from sensorflow.evaluation.process_units import usage_summary
72
+ store = get_store()
73
+ ds = store.get("datasets", dataset_id)
74
+ if ds:
75
+ dataset_version = ds.version
76
+ drill["dataset_ids"] = [dataset_id]
77
+ else:
78
+ placeholders.append(f"dataset_missing:{dataset_id}")
79
+ summary = usage_summary(store)
80
+ compute_usage = [summary] if isinstance(summary, dict) else list(summary or [])
81
+ except Exception as exc:
82
+ placeholders.append(f"labeleval_error:{exc}")
83
+ compute_usage = []
84
+ else:
85
+ placeholders.append("dataset_id")
86
+
87
+ if include_gates:
88
+ try:
89
+ from sensorflow.platform.gates import evaluate_multi_gates, load_gate_config
90
+ cfg = load_gate_config()
91
+ thresholds = {k: v.get("thresholds", {}) for k, v in cfg.get("gates", {}).items()}
92
+ gate_payload = evaluate_multi_gates(
93
+ sequence_id=sequence_id,
94
+ candidate_run_id=candidate_run_id or run_id,
95
+ baseline_run_id=baseline_run_id,
96
+ )
97
+ gate_results = gate_payload.get("gates") or []
98
+ except Exception as exc:
99
+ placeholders.append(f"gates_error:{exc}")
100
+
101
+ if not metrics:
102
+ placeholders.append("metrics")
103
+ if not model_versions:
104
+ placeholders.append("model_versions")
105
+ model_versions = ["placeholder-model"]
106
+ if not compute_usage:
107
+ compute_usage = [{"stage": "platform_evidence", "units": 0, "note": "placeholder"}]
108
+ placeholders.append("compute_usage_detail")
109
+
110
+ return EvidencePackage(
111
+ package_id=_id("evidence"),
112
+ evaluation_id=eval_id,
113
+ dataset_id=dataset_id,
114
+ dataset_version=dataset_version,
115
+ population_id=population_id,
116
+ model_versions=model_versions,
117
+ label_version=label_version,
118
+ metrics=metrics,
119
+ thresholds=thresholds,
120
+ gate_results=gate_results,
121
+ compute_usage=compute_usage,
122
+ drill_down_ids=drill,
123
+ placeholders=sorted(set(placeholders)),
124
+ provenance=Provenance(
125
+ source_system="platform",
126
+ notes="Phase 1 evidence stub — fill placeholders in later phases",
127
+ parent_ids=[x for x in [run_id, dataset_id, sequence_id] if x],
128
+ ),
129
+ )
130
+
131
+
132
+ def export_evidence_package(package: EvidencePackage, path: Optional[Path] = None) -> Path:
133
+ EVIDENCE_DIR.mkdir(parents=True, exist_ok=True)
134
+ path = path or (EVIDENCE_DIR / f"{package.package_id}.json")
135
+ # Append-only: never overwrite an existing package id silently
136
+ if path.exists():
137
+ path = EVIDENCE_DIR / f"{package.package_id}-{_id('rev')}.json"
138
+ with open(path, "w") as f:
139
+ json.dump(package.model_dump(), f, indent=2)
140
+ return path
sensorflow/platform/gates.py ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Multi-gate readiness skeleton — config-driven thresholds.
2
+
3
+ Extends existing QualityGate + LaunchGateEvaluator toward:
4
+ Scenario | Coverage | Regression | Safety | Release
5
+
6
+ Phases 4–6 fill Safety/ODD/Release deeply; Phase 1 returns honest ready=False
7
+ for unwired gates and real results for quality/launch when sequence artifacts exist.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ from pathlib import Path
14
+ from typing import Any, Dict, List, Optional
15
+
16
+ from sensorflow.platform.entities import GateResult, QualityGateDef, Provenance, _id
17
+
18
+ DEFAULT_GATE_CONFIG_PATH = Path("runs/platform/gate_config.json")
19
+
20
+ # Config-driven defaults — never hard-code safety cutoffs in callers.
21
+ DEFAULT_GATE_CONFIG: Dict[str, Any] = {
22
+ "schema_version": "platform-gates-1.0",
23
+ "gates": {
24
+ "scenario": {
25
+ "enabled": True,
26
+ "ready": False, # Phase 5/6
27
+ "thresholds": {"min_scenario_types_covered": 5},
28
+ "message": "TODO(Phase 5): scenario coverage vs ODDDefinition",
29
+ },
30
+ "coverage": {
31
+ "enabled": True,
32
+ "ready": False,
33
+ "thresholds": {"min_cohort_support": 200, "min_dim_coverage": 0.8},
34
+ "message": "TODO(Phase 5): ODD combinatorial coverage via MegaEval cube",
35
+ },
36
+ "regression": {
37
+ "enabled": True,
38
+ "ready": True, # can use MegaEval compare / LabelEval regression when run_ids provided
39
+ "thresholds": {
40
+ "max_recall_drop": 0.010,
41
+ "max_precision_drop": 0.015,
42
+ "max_safety_recall_drop": 0.005,
43
+ },
44
+ "message": "Uses MegaEval promotion policy when candidate/baseline runs supplied",
45
+ },
46
+ "safety": {
47
+ "enabled": True,
48
+ "ready": False,
49
+ "thresholds": {}, # filled in Phase 4 from config only
50
+ "message": "TODO(Phase 4): SSAM DRAC/DeltaS conflict engine",
51
+ },
52
+ "release": {
53
+ "enabled": True,
54
+ "ready": False,
55
+ "thresholds": {"require_all_prior_passed": 1.0},
56
+ "message": "TODO(Phase 6): release orchestration after Safety+Regression",
57
+ },
58
+ "quality": {
59
+ "enabled": True,
60
+ "ready": True,
61
+ "thresholds": {
62
+ "map_3d": 0.65,
63
+ "orientation_error_deg": 5.0,
64
+ "id_swap_rate": 0.02,
65
+ "track_fragmentation_rate": 0.05,
66
+ "position_error_m": 2.0,
67
+ },
68
+ "message": "Delegates to sensorflow.quality_gate.QualityGate when sequence_id set",
69
+ },
70
+ "launch": {
71
+ "enabled": True,
72
+ "ready": True,
73
+ "thresholds": {
74
+ "map_3d": 0.65,
75
+ "orientation_error_deg": 5.0,
76
+ "id_swap_rate": 0.02,
77
+ "track_fragmentation_rate": 0.05,
78
+ },
79
+ "message": "Delegates to LaunchGateEvaluator when sequence_id set",
80
+ },
81
+ },
82
+ }
83
+
84
+
85
+ def load_gate_config(path: Optional[Path] = None) -> Dict[str, Any]:
86
+ path = path or DEFAULT_GATE_CONFIG_PATH
87
+ if path.exists():
88
+ with open(path) as f:
89
+ cfg = json.load(f)
90
+ # merge missing keys from defaults without wiping user thresholds
91
+ merged = json.loads(json.dumps(DEFAULT_GATE_CONFIG))
92
+ for gname, gcfg in (cfg.get("gates") or {}).items():
93
+ if gname in merged["gates"]:
94
+ merged["gates"][gname].update(gcfg)
95
+ else:
96
+ merged["gates"][gname] = gcfg
97
+ return merged
98
+ path.parent.mkdir(parents=True, exist_ok=True)
99
+ with open(path, "w") as f:
100
+ json.dump(DEFAULT_GATE_CONFIG, f, indent=2)
101
+ return json.loads(json.dumps(DEFAULT_GATE_CONFIG))
102
+
103
+
104
+ def save_gate_config(cfg: Dict[str, Any], path: Optional[Path] = None) -> Path:
105
+ path = path or DEFAULT_GATE_CONFIG_PATH
106
+ path.parent.mkdir(parents=True, exist_ok=True)
107
+ with open(path, "w") as f:
108
+ json.dump(cfg, f, indent=2)
109
+ return path
110
+
111
+
112
+ GATE_ORDER = ["scenario", "coverage", "regression", "safety", "quality", "launch", "release"]
113
+
114
+
115
+ def list_gate_defs(cfg: Optional[Dict[str, Any]] = None) -> List[QualityGateDef]:
116
+ cfg = cfg or load_gate_config()
117
+ out = []
118
+ for name in GATE_ORDER:
119
+ g = cfg["gates"].get(name, {})
120
+ out.append(QualityGateDef(
121
+ gate_id=f"gate-{name}",
122
+ name=name.title(),
123
+ gate_type=name, # type: ignore[arg-type]
124
+ thresholds=dict(g.get("thresholds") or {}),
125
+ config_path=str(DEFAULT_GATE_CONFIG_PATH),
126
+ enabled=bool(g.get("enabled", True)),
127
+ version="v1",
128
+ provenance=Provenance(source_system="platform", notes=g.get("message", "")),
129
+ ))
130
+ return out
131
+
132
+
133
+ def evaluate_multi_gates(
134
+ *,
135
+ sequence_id: Optional[str] = None,
136
+ candidate_run_id: Optional[str] = None,
137
+ baseline_run_id: Optional[str] = None,
138
+ config_path: Optional[Path] = None,
139
+ ) -> Dict[str, Any]:
140
+ cfg = load_gate_config(config_path)
141
+ results: List[GateResult] = []
142
+
143
+ for name in GATE_ORDER:
144
+ g = cfg["gates"][name]
145
+ if not g.get("enabled", True):
146
+ continue
147
+ ready = bool(g.get("ready", False))
148
+ thresholds = dict(g.get("thresholds") or {})
149
+ result = GateResult(
150
+ result_id=_id("gateres"),
151
+ gate_id=f"gate-{name}",
152
+ gate_type=name,
153
+ thresholds=thresholds,
154
+ ready=ready,
155
+ message=g.get("message", ""),
156
+ scope_ref=sequence_id or candidate_run_id,
157
+ passed=None,
158
+ )
159
+
160
+ if name == "quality" and sequence_id and ready:
161
+ result = _eval_quality(sequence_id, thresholds, result)
162
+ elif name == "launch" and sequence_id and ready:
163
+ result = _eval_launch(sequence_id, result)
164
+ elif name == "regression" and candidate_run_id and baseline_run_id and ready:
165
+ result = _eval_regression(candidate_run_id, baseline_run_id, thresholds, result)
166
+ elif not ready:
167
+ result.passed = None
168
+ result.message = g.get("message") or f"{name} gate skeleton — not yet wired"
169
+ else:
170
+ result.passed = None
171
+ result.message = f"{name} gate ready but missing required inputs"
172
+
173
+ results.append(result)
174
+
175
+ # Overall readiness: all ready gates that produced a boolean must pass;
176
+ # unwired gates block "release ready" but do not fail the suite silently.
177
+ wired = [r for r in results if r.ready and r.passed is not None]
178
+ unwired = [r for r in results if not r.ready]
179
+ all_wired_passed = all(r.passed for r in wired) if wired else False
180
+
181
+ return {
182
+ "config_path": str(config_path or DEFAULT_GATE_CONFIG_PATH),
183
+ "sequence_id": sequence_id,
184
+ "candidate_run_id": candidate_run_id,
185
+ "baseline_run_id": baseline_run_id,
186
+ "gates": [r.model_dump() for r in results],
187
+ "summary": {
188
+ "wired_count": len(wired),
189
+ "unwired_count": len(unwired),
190
+ "wired_all_passed": all_wired_passed,
191
+ "release_ready": False, # Phase 6
192
+ "unwired_gate_types": [r.gate_type for r in unwired],
193
+ },
194
+ }
195
+
196
+
197
+ def _eval_quality(sequence_id: str, thresholds: Dict[str, float], result: GateResult) -> GateResult:
198
+ try:
199
+ from sensorflow.quality_gate import QualityGate
200
+ from sensorflow.schemas.unified_frame import UnifiedSequence
201
+
202
+ manifest = Path("runs/pipeline") / sequence_id / "manifest.json"
203
+ tracks_path = Path("runs/pipeline") / sequence_id / "tracks.json"
204
+ if not manifest.exists() or not tracks_path.exists():
205
+ result.passed = False
206
+ result.failures = [{"metric": "artifacts", "message": "manifest/tracks missing"}]
207
+ result.message = "Quality gate: run ingest/auto-label/track first"
208
+ return result
209
+ sequence = UnifiedSequence.load(manifest)
210
+ pred_tracks = json.loads(tracks_path.read_text())
211
+ gate = QualityGate(thresholds=thresholds)
212
+ out = gate.evaluate(sequence, pred_tracks)
213
+ result.passed = bool(out.get("passed"))
214
+ result.metrics = out.get("metric_card") or {}
215
+ result.failures = out.get("quality_report", {}).get("failures") or out.get("failures") or []
216
+ result.message = "QualityGate evaluated"
217
+ except Exception as exc:
218
+ result.passed = False
219
+ result.failures = [{"error": str(exc)}]
220
+ result.message = f"Quality gate error: {exc}"
221
+ return result
222
+
223
+
224
+ def _eval_launch(sequence_id: str, result: GateResult) -> GateResult:
225
+ try:
226
+ from sensorflow.launch_gate_evaluator import LaunchGateEvaluator
227
+ evaluator = LaunchGateEvaluator()
228
+ out = evaluator.evaluate(sequence_id)
229
+ result.passed = bool(out.get("passed"))
230
+ result.failures = out.get("failures") or []
231
+ result.metrics = out.get("metrics") or {}
232
+ result.thresholds = evaluator.thresholds
233
+ result.message = "LaunchGateEvaluator evaluated"
234
+ except Exception as exc:
235
+ result.passed = False
236
+ result.failures = [{"error": str(exc)}]
237
+ result.message = f"Launch gate error: {exc}"
238
+ return result
239
+
240
+
241
+ def _eval_regression(
242
+ candidate_run_id: str,
243
+ baseline_run_id: str,
244
+ thresholds: Dict[str, float],
245
+ result: GateResult,
246
+ ) -> GateResult:
247
+ try:
248
+ from sensorflow.platform.compare import compare_models
249
+ cmp = compare_models(
250
+ [baseline_run_id, candidate_run_id],
251
+ baseline_run_id=baseline_run_id,
252
+ policy=thresholds,
253
+ )
254
+ pair = (cmp.get("pairwise") or [{}])[0]
255
+ rec = pair.get("recommendation")
256
+ result.passed = rec == "PROMOTE"
257
+ result.failures = [{"blocker": b} for b in (pair.get("blockers") or [])]
258
+ result.metrics = {"recommendation": rec, "headline": pair.get("headline")}
259
+ result.message = f"Regression gate: {rec}"
260
+ except Exception as exc:
261
+ result.passed = False
262
+ result.failures = [{"error": str(exc)}]
263
+ result.message = f"Regression gate error: {exc}"
264
+ return result
sensorflow/platform/levels.py ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Evaluation abstraction across aggregate levels.
2
+
3
+ Unifies LabelEval (frame/annotation) and MegaEval (container/cohort/population)
4
+ into one ladder without duplicating storage.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from enum import Enum
10
+ from typing import Any, Dict, List, Optional
11
+
12
+ from pydantic import BaseModel, Field
13
+
14
+
15
+ class AggregateLevel(str, Enum):
16
+ FRAME = "frame"
17
+ CLIP = "clip" # sequence / short clip
18
+ SCENE = "scene"
19
+ DRIVE = "drive"
20
+ CONTAINER = "container"
21
+ DATASET = "dataset"
22
+ COHORT = "cohort"
23
+ POPULATION = "population"
24
+
25
+ @classmethod
26
+ def ladder(cls) -> List["AggregateLevel"]:
27
+ return [
28
+ cls.FRAME,
29
+ cls.CLIP,
30
+ cls.SCENE,
31
+ cls.DRIVE,
32
+ cls.CONTAINER,
33
+ cls.DATASET,
34
+ cls.COHORT,
35
+ cls.POPULATION,
36
+ ]
37
+
38
+ def parent(self) -> Optional["AggregateLevel"]:
39
+ ladder = self.ladder()
40
+ i = ladder.index(self)
41
+ return ladder[i + 1] if i + 1 < len(ladder) else None
42
+
43
+ def child(self) -> Optional["AggregateLevel"]:
44
+ ladder = self.ladder()
45
+ i = ladder.index(self)
46
+ return ladder[i - 1] if i > 0 else None
47
+
48
+
49
+ # Which backend owns primary aggregates at each level
50
+ LEVEL_BACKEND: Dict[AggregateLevel, str] = {
51
+ AggregateLevel.FRAME: "labeleval",
52
+ AggregateLevel.CLIP: "labeleval",
53
+ AggregateLevel.SCENE: "labeleval",
54
+ AggregateLevel.DRIVE: "platform", # stub until Phase 5/6
55
+ AggregateLevel.CONTAINER: "megaeval",
56
+ AggregateLevel.DATASET: "labeleval",
57
+ AggregateLevel.COHORT: "megaeval",
58
+ AggregateLevel.POPULATION: "megaeval",
59
+ }
60
+
61
+
62
+ class EvaluationScope(BaseModel):
63
+ """Points at one aggregate cell without inventing a second DB."""
64
+
65
+ level: AggregateLevel
66
+ evaluation_id: Optional[str] = None
67
+ dataset_id: Optional[str] = None
68
+ population_id: Optional[str] = None
69
+ run_id: Optional[str] = None
70
+ container_id: Optional[str] = None
71
+ mega_container_id: Optional[int] = None
72
+ drive_id: Optional[str] = None
73
+ scene_id: Optional[str] = None
74
+ sequence_id: Optional[str] = None
75
+ frame_id: Optional[str] = None
76
+ cohort_filters: Dict[str, List[str]] = Field(default_factory=dict)
77
+ model_version: Optional[str] = None
78
+ label_version: Optional[str] = None
79
+
80
+ def backend(self) -> str:
81
+ return LEVEL_BACKEND[self.level]
82
+
83
+ def drill_down_hint(self) -> AggregateLevel:
84
+ child = self.level.child()
85
+ return child or self.level
86
+
87
+
88
+ def summarize_scope(scope: EvaluationScope) -> Dict[str, Any]:
89
+ return {
90
+ "level": scope.level.value,
91
+ "backend": scope.backend(),
92
+ "parent_level": scope.level.parent().value if scope.level.parent() else None,
93
+ "child_level": scope.level.child().value if scope.level.child() else None,
94
+ "refs": {
95
+ k: v
96
+ for k, v in {
97
+ "evaluation_id": scope.evaluation_id,
98
+ "dataset_id": scope.dataset_id,
99
+ "population_id": scope.population_id,
100
+ "run_id": scope.run_id,
101
+ "container_id": scope.container_id,
102
+ "mega_container_id": scope.mega_container_id,
103
+ "drive_id": scope.drive_id,
104
+ "scene_id": scope.scene_id,
105
+ "sequence_id": scope.sequence_id,
106
+ "frame_id": scope.frame_id,
107
+ "model_version": scope.model_version,
108
+ }.items()
109
+ if v is not None
110
+ },
111
+ "cohort_filters": scope.cohort_filters,
112
+ }
sensorflow/platform/metrics_engine.py ADDED
@@ -0,0 +1,163 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Metric engine: precision/recall/F1/IoU + verification rates.
2
+
3
+ Pure functions — no I/O. Callers feed counts from MegaEval containers or
4
+ LabelEval match stats.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from typing import Any, Dict, Iterable, List, Optional, Sequence
10
+
11
+
12
+ def safe_div(num: float, den: float) -> Optional[float]:
13
+ if den <= 0:
14
+ return None
15
+ return num / den
16
+
17
+
18
+ def round4(v: Optional[float]) -> Optional[float]:
19
+ return None if v is None else round(float(v), 4)
20
+
21
+
22
+ def precision_recall_f1(tp: int, fp: int, fn: int) -> Dict[str, Optional[float]]:
23
+ precision = safe_div(tp, tp + fp)
24
+ recall = safe_div(tp, tp + fn)
25
+ f1 = None
26
+ if precision is not None and recall is not None and (precision + recall) > 0:
27
+ f1 = 2 * precision * recall / (precision + recall)
28
+ return {
29
+ "precision": round4(precision),
30
+ "recall": round4(recall),
31
+ "f1": round4(f1),
32
+ "tp": tp,
33
+ "fp": fp,
34
+ "fn": fn,
35
+ }
36
+
37
+
38
+ def mean_iou(sum_iou: float, tp: int) -> Optional[float]:
39
+ return round4(safe_div(sum_iou, tp))
40
+
41
+
42
+ def verification_rates(
43
+ *,
44
+ n_objects: int,
45
+ verified: int = 0,
46
+ reviewed: int = 0,
47
+ disputed: int = 0,
48
+ auto_accepted: int = 0,
49
+ hitl: int = 0,
50
+ unverified: Optional[int] = None,
51
+ ) -> Dict[str, Any]:
52
+ """Verification / HITL funnel rates at container (or dataset) scope.
53
+
54
+ unverified defaults to n_objects - verified when not provided.
55
+ """
56
+ n = max(0, int(n_objects))
57
+ verified = max(0, int(verified))
58
+ reviewed = max(0, int(reviewed))
59
+ disputed = max(0, int(disputed))
60
+ auto_accepted = max(0, int(auto_accepted))
61
+ hitl = max(0, int(hitl))
62
+ if unverified is None:
63
+ unverified = max(0, n - verified)
64
+
65
+ def rate(count: int) -> Optional[float]:
66
+ return round4(safe_div(count, n)) if n > 0 else None
67
+
68
+ return {
69
+ "n_objects": n,
70
+ "verified": verified,
71
+ "unverified": unverified,
72
+ "disputed": disputed,
73
+ "auto_accepted": auto_accepted,
74
+ "hitl": hitl,
75
+ "reviewed": reviewed,
76
+ "verified_rate": rate(verified),
77
+ "unverified_rate": rate(unverified),
78
+ "disputed_rate": rate(disputed),
79
+ "auto_accept_rate": rate(auto_accepted),
80
+ "hitl_rate": rate(hitl),
81
+ "review_coverage": rate(reviewed),
82
+ }
83
+
84
+
85
+ def container_quality_metrics(
86
+ *,
87
+ tp: int,
88
+ fp: int,
89
+ fn: int,
90
+ sum_iou: float = 0.0,
91
+ n_objects: int = 0,
92
+ verified: int = 0,
93
+ reviewed: int = 0,
94
+ disputed: int = 0,
95
+ auto_accepted: int = 0,
96
+ hitl: int = 0,
97
+ anomalies: int = 0,
98
+ ) -> Dict[str, Any]:
99
+ prf = precision_recall_f1(tp, fp, fn)
100
+ n = n_objects or (tp + fp)
101
+ ver = verification_rates(
102
+ n_objects=n,
103
+ verified=verified,
104
+ reviewed=reviewed,
105
+ disputed=disputed,
106
+ auto_accepted=auto_accepted,
107
+ hitl=hitl,
108
+ )
109
+ return {
110
+ **prf,
111
+ "mean_iou": mean_iou(sum_iou, tp),
112
+ "anomalies": anomalies,
113
+ "anomaly_rate": round4(safe_div(anomalies, n)) if n else None,
114
+ "verification": ver,
115
+ }
116
+
117
+
118
+ def aggregate_container_rows(rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]:
119
+ """Roll up a page of container metric dicts into one profile summary."""
120
+ tp = fp = fn = verified = reviewed = disputed = auto_acc = hitl = anomalies = n = 0
121
+ sum_iou = 0.0
122
+ for r in rows:
123
+ tp += int(r.get("tp") or 0)
124
+ fp += int(r.get("fp") or 0)
125
+ fn += int(r.get("fn") or 0)
126
+ verified += int(r.get("verified") or 0)
127
+ reviewed += int(r.get("reviewed") or 0)
128
+ disputed += int(r.get("disputed") or 0)
129
+ auto_acc += int(r.get("auto_accepted") or 0)
130
+ hitl += int(r.get("hitl") or 0)
131
+ anomalies += int(r.get("anomalies") or 0)
132
+ n += int(r.get("n_objects") or 0)
133
+ mi = r.get("mean_iou")
134
+ tpi = int(r.get("tp") or 0)
135
+ if mi is not None and tpi > 0:
136
+ sum_iou += float(mi) * tpi
137
+ elif r.get("sum_iou") is not None:
138
+ sum_iou += float(r["sum_iou"])
139
+ return container_quality_metrics(
140
+ tp=tp, fp=fp, fn=fn, sum_iou=sum_iou, n_objects=n,
141
+ verified=verified, reviewed=reviewed, disputed=disputed,
142
+ auto_accepted=auto_acc, hitl=hitl, anomalies=anomalies,
143
+ )
144
+
145
+
146
+ def delta_metrics(
147
+ baseline: Dict[str, Optional[float]],
148
+ candidate: Dict[str, Optional[float]],
149
+ keys: Optional[Iterable[str]] = None,
150
+ ) -> List[Dict[str, Any]]:
151
+ keys = list(keys or ("precision", "recall", "f1", "mean_iou", "safety_recall", "anomaly_rate"))
152
+ out = []
153
+ for k in keys:
154
+ b, c = baseline.get(k), candidate.get(k)
155
+ if b is None or c is None:
156
+ continue
157
+ out.append({
158
+ "metric": k,
159
+ "baseline": round4(float(b)),
160
+ "candidate": round4(float(c)),
161
+ "delta": round4(float(c) - float(b)),
162
+ })
163
+ return out
src/components/platform/ContainerQualityPanel.tsx ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Container quality cards/table with verification rates (Phase 1 foundation).
3
+ * Extends MegaEval ContainersTab data via /api/containers/quality.
4
+ */
5
+ import Box from '@mui/material/Box';
6
+ import Table from '@mui/material/Table';
7
+ import TableBody from '@mui/material/TableBody';
8
+ import TableCell from '@mui/material/TableCell';
9
+ import TableHead from '@mui/material/TableHead';
10
+ import TableRow from '@mui/material/TableRow';
11
+ import Typography from '@mui/material/Typography';
12
+ import { useContainerQuality } from '../../services/platform';
13
+ import { ErrorNote, LoadingBox, MetricCard, SectionCard, fmtPct } from '../labeleval/shared';
14
+
15
+ export default function ContainerQualityPanel({
16
+ runId,
17
+ refreshKey = 0,
18
+ }: {
19
+ runId: string | null;
20
+ refreshKey?: number;
21
+ }) {
22
+ const poll = useContainerQuality(runId, refreshKey);
23
+
24
+ if (!runId) {
25
+ return (
26
+ <SectionCard title="Container quality">
27
+ <Typography variant="body2" color="text.secondary">
28
+ Select a published evaluation run to load container quality profiles.
29
+ </Typography>
30
+ </SectionCard>
31
+ );
32
+ }
33
+ if (poll.error && !poll.data) return <ErrorNote error={poll.error} />;
34
+ if (poll.loading && !poll.data) return <LoadingBox label="Loading container quality…" />;
35
+ const data = poll.data;
36
+ if (!data) return null;
37
+
38
+ const v = data.verification_headline || data.summary?.verification;
39
+
40
+ return (
41
+ <Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
42
+ <SectionCard
43
+ title="Container quality profile"
44
+ action={
45
+ <Typography variant="caption" color="text.secondary">
46
+ {data.total} containers · {data.model_version}
47
+ </Typography>
48
+ }
49
+ >
50
+ <Box sx={{ display: 'flex', gap: 1.5, flexWrap: 'wrap', mb: 2 }}>
51
+ <MetricCard label="Precision" value={fmtPct(data.summary.precision)} />
52
+ <MetricCard label="Recall" value={fmtPct(data.summary.recall)} />
53
+ <MetricCard label="F1" value={fmtPct(data.summary.f1)} />
54
+ <MetricCard label="Mean IoU" value={fmtPct(data.summary.mean_iou)} />
55
+ <MetricCard label="Verified" value={fmtPct(v?.verified_rate)} sub={`${v?.verified ?? 0} labels`} />
56
+ <MetricCard label="HITL rate" value={fmtPct(v?.hitl_rate)} />
57
+ <MetricCard label="Unverified" value={fmtPct(v?.unverified_rate)} />
58
+ <MetricCard label="Disputed" value={fmtPct(v?.disputed_rate)} sub="Phase 2 hook" />
59
+ </Box>
60
+
61
+ <Table size="small">
62
+ <TableHead>
63
+ <TableRow>
64
+ <TableCell>Container</TableCell>
65
+ <TableCell>Status</TableCell>
66
+ <TableCell align="right">P</TableCell>
67
+ <TableCell align="right">R</TableCell>
68
+ <TableCell align="right">F1</TableCell>
69
+ <TableCell align="right">IoU</TableCell>
70
+ <TableCell align="right">Verified</TableCell>
71
+ <TableCell align="right">HITL</TableCell>
72
+ <TableCell align="right">Risk</TableCell>
73
+ </TableRow>
74
+ </TableHead>
75
+ <TableBody>
76
+ {data.containers.slice(0, 20).map((c) => (
77
+ <TableRow key={c.container_id} hover>
78
+ <TableCell>#{c.container_id}</TableCell>
79
+ <TableCell>{c.status ?? '—'}</TableCell>
80
+ <TableCell align="right">{fmtPct(c.precision)}</TableCell>
81
+ <TableCell align="right">{fmtPct(c.recall)}</TableCell>
82
+ <TableCell align="right">{fmtPct(c.f1)}</TableCell>
83
+ <TableCell align="right">{fmtPct(c.mean_iou)}</TableCell>
84
+ <TableCell align="right">{fmtPct(c.verification?.verified_rate)}</TableCell>
85
+ <TableCell align="right">{fmtPct(c.verification?.hitl_rate)}</TableCell>
86
+ <TableCell align="right">{c.risk_score.toFixed(2)}</TableCell>
87
+ </TableRow>
88
+ ))}
89
+ </TableBody>
90
+ </Table>
91
+ </SectionCard>
92
+ </Box>
93
+ );
94
+ }
src/components/platform/GateStatusPanel.tsx ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Multi-gate readiness skeleton: Scenario / Coverage / Regression / Safety / Release
3
+ * (+ quality / launch when wired).
4
+ */
5
+ import Box from '@mui/material/Box';
6
+ import Chip from '@mui/material/Chip';
7
+ import Typography from '@mui/material/Typography';
8
+ import { CheckCircle2, CircleDashed, XCircle } from 'lucide-react';
9
+ import { useGateStatus } from '../../services/platform';
10
+ import { ErrorNote, LoadingBox, SectionCard } from '../labeleval/shared';
11
+
12
+ const DISPLAY_ORDER = ['scenario', 'coverage', 'regression', 'safety', 'quality', 'launch', 'release'];
13
+
14
+ function statusChip(passed: boolean | null, ready: boolean) {
15
+ if (!ready) {
16
+ return <Chip size="small" icon={<CircleDashed size={14} />} label="Skeleton" variant="outlined" />;
17
+ }
18
+ if (passed === true) {
19
+ return <Chip size="small" color="success" icon={<CheckCircle2 size={14} />} label="Pass" />;
20
+ }
21
+ if (passed === false) {
22
+ return <Chip size="small" color="error" icon={<XCircle size={14} />} label="Fail" />;
23
+ }
24
+ return <Chip size="small" label="Pending inputs" variant="outlined" />;
25
+ }
26
+
27
+ export default function GateStatusPanel({
28
+ candidateRunId,
29
+ baselineRunId,
30
+ refreshKey = 0,
31
+ }: {
32
+ candidateRunId: string | null;
33
+ baselineRunId: string | null;
34
+ refreshKey?: number;
35
+ }) {
36
+ const poll = useGateStatus(candidateRunId, baselineRunId, refreshKey);
37
+
38
+ if (poll.error && !poll.data) return <ErrorNote error={poll.error} />;
39
+ if (poll.loading && !poll.data) return <LoadingBox label="Loading gate status…" />;
40
+ const data = poll.data;
41
+ if (!data) return null;
42
+
43
+ const byType = Object.fromEntries(data.gates.map((g) => [g.gate_type, g]));
44
+
45
+ return (
46
+ <SectionCard
47
+ title="Multi-gate readiness"
48
+ action={
49
+ <Typography variant="caption" color="text.secondary">
50
+ {data.summary.release_ready
51
+ ? 'Release ready'
52
+ : `${data.summary.unwired_count} gates still Phase 2–6 stubs`}
53
+ </Typography>
54
+ }
55
+ >
56
+ <Box sx={{ display: 'flex', flexDirection: 'column', gap: 1 }}>
57
+ {DISPLAY_ORDER.map((type) => {
58
+ const g = byType[type];
59
+ if (!g) return null;
60
+ return (
61
+ <Box
62
+ key={type}
63
+ sx={{
64
+ display: 'flex',
65
+ alignItems: 'center',
66
+ gap: 1.5,
67
+ py: 0.75,
68
+ borderBottom: '1px solid #232a31',
69
+ }}
70
+ >
71
+ <Typography sx={{ minWidth: 110, fontWeight: 600, textTransform: 'capitalize' }}>
72
+ {type}
73
+ </Typography>
74
+ {statusChip(g.passed, g.ready)}
75
+ <Typography variant="body2" color="text.secondary" sx={{ flex: 1 }}>
76
+ {g.message}
77
+ </Typography>
78
+ </Box>
79
+ );
80
+ })}
81
+ </Box>
82
+ </SectionCard>
83
+ );
84
+ }
src/components/platform/ModelComparePanel.tsx ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Multi-model compare panel (A vs B vs C) — wraps /api/models/compare.
3
+ */
4
+ import { useMemo } from 'react';
5
+ import Alert from '@mui/material/Alert';
6
+ import Box from '@mui/material/Box';
7
+ import Table from '@mui/material/Table';
8
+ import TableBody from '@mui/material/TableBody';
9
+ import TableCell from '@mui/material/TableCell';
10
+ import TableHead from '@mui/material/TableHead';
11
+ import TableRow from '@mui/material/TableRow';
12
+ import Typography from '@mui/material/Typography';
13
+ import { compareModels } from '../../services/platform';
14
+ import { usePoll } from '../../services/labeleval';
15
+ import { ErrorNote, LoadingBox, SectionCard, fmtPct } from '../labeleval/shared';
16
+
17
+ export default function ModelComparePanel({
18
+ runId,
19
+ baselineRunId,
20
+ extraRunIds = [],
21
+ refreshKey = 0,
22
+ }: {
23
+ runId: string | null;
24
+ baselineRunId: string | null;
25
+ extraRunIds?: string[];
26
+ refreshKey?: number;
27
+ }) {
28
+ const runIds = useMemo(() => {
29
+ const ids = [baselineRunId, runId, ...extraRunIds].filter(Boolean) as string[];
30
+ return [...new Set(ids)];
31
+ }, [runId, baselineRunId, extraRunIds]);
32
+
33
+ const cmp = usePoll(
34
+ () =>
35
+ runIds.length >= 2
36
+ ? compareModels(runIds, baselineRunId)
37
+ : Promise.resolve(null),
38
+ null,
39
+ [runIds.join(','), baselineRunId, refreshKey]
40
+ );
41
+
42
+ if (runIds.length < 2) {
43
+ return (
44
+ <Alert severity="info" variant="outlined">
45
+ Select a baseline run (and optionally more candidates) to compare models A / B / C.
46
+ </Alert>
47
+ );
48
+ }
49
+ if (cmp.error && !cmp.data) return <ErrorNote error={cmp.error} />;
50
+ if (cmp.loading && !cmp.data) return <LoadingBox label="Comparing models…" />;
51
+ const data = cmp.data;
52
+ if (!data) return null;
53
+
54
+ const modelNames = data.models.map((m) => m.model_version);
55
+
56
+ return (
57
+ <Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
58
+ <SectionCard
59
+ title="Model compare (A / B / C)"
60
+ action={
61
+ <Typography variant="caption" color="text.secondary">
62
+ Baseline {data.baseline_run_id.slice(0, 8)}…
63
+ </Typography>
64
+ }
65
+ >
66
+ <Box sx={{ display: 'flex', gap: 1, flexWrap: 'wrap', mb: 2 }}>
67
+ {data.recommendations.map((r) => (
68
+ <Alert
69
+ key={r.candidate}
70
+ severity={r.recommendation === 'PROMOTE' ? 'success' : 'warning'}
71
+ variant="outlined"
72
+ sx={{ py: 0 }}
73
+ >
74
+ {r.candidate}: {r.recommendation}
75
+ {r.blocker_count ? ` (${r.blocker_count} blockers)` : ''}
76
+ </Alert>
77
+ ))}
78
+ </Box>
79
+
80
+ <Table size="small">
81
+ <TableHead>
82
+ <TableRow>
83
+ <TableCell>Metric</TableCell>
84
+ {modelNames.map((m) => (
85
+ <TableCell key={m} align="right">
86
+ {m}
87
+ </TableCell>
88
+ ))}
89
+ </TableRow>
90
+ </TableHead>
91
+ <TableBody>
92
+ {data.metric_matrix.map((row) => (
93
+ <TableRow key={row.metric}>
94
+ <TableCell>{row.metric}</TableCell>
95
+ {modelNames.map((m) => {
96
+ const v = row[m] as number | null | undefined;
97
+ const d = row.deltas_vs_baseline?.[m];
98
+ return (
99
+ <TableCell key={m} align="right">
100
+ {fmtPct(v ?? null)}
101
+ {d !== undefined && d !== null ? (
102
+ <Typography component="span" variant="caption" sx={{ ml: 0.5, color: d < 0 ? '#c62828' : '#2e7d32' }}>
103
+ {d >= 0 ? '+' : ''}
104
+ {(d * 100).toFixed(1)}pp
105
+ </Typography>
106
+ ) : null}
107
+ </TableCell>
108
+ );
109
+ })}
110
+ </TableRow>
111
+ ))}
112
+ </TableBody>
113
+ </Table>
114
+ </SectionCard>
115
+ </Box>
116
+ );
117
+ }
src/pages/CommandCenterPage.tsx CHANGED
@@ -40,6 +40,9 @@ import CompareTab from '../components/megaeval/CompareTab';
40
  import ReviewTab from '../components/megaeval/ReviewTab';
41
  import ShiftTab from '../components/megaeval/ShiftTab';
42
  import LineageCard from '../components/megaeval/LineageCard';
 
 
 
43
 
44
  const ACTIVE_STATUSES: RunStatus[] = ['created', 'queued', 'running', 'reducing', 'materializing'];
45
 
@@ -49,6 +52,7 @@ type TabId =
49
  | 'containers'
50
  | 'investigation'
51
  | 'compare'
 
52
  | 'review'
53
  | 'shift'
54
  | 'lineage';
@@ -290,28 +294,49 @@ export default function CommandCenterPage() {
290
  <Tab label="Containers" value="containers" />
291
  <Tab label="Investigation" value="investigation" />
292
  <Tab label="Compare" value="compare" />
 
293
  <Tab label="Review" value="review" />
294
  <Tab label="Shift" value="shift" />
295
  <Tab label="Lineage" value="lineage" />
296
  </Tabs>
297
 
298
  {tab === 'quality' ? (
299
- <QualityTab runId={selectedRun.run_id} runs={runs} refreshKey={publishTick} />
 
 
 
300
  ) : null}
301
  {tab === 'cohorts' ? <CohortTab runId={selectedRun.run_id} refreshKey={publishTick} /> : null}
302
  {tab === 'containers' ? (
303
- <ContainersTab
304
- runId={selectedRun.run_id}
305
- refreshKey={publishTick}
306
- drillContainerId={drillContainerId}
307
- onDrill={setDrillContainerId}
308
- />
 
 
 
309
  ) : null}
310
  {tab === 'investigation' ? (
311
  <InvestigationTab runId={selectedRun.run_id} onOpenContainer={openContainer} />
312
  ) : null}
313
  {tab === 'compare' ? (
314
- <CompareTab runId={selectedRun.run_id} baselineRunId={baselineRunId} refreshKey={publishTick} />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
315
  ) : null}
316
  {tab === 'review' ? (
317
  <ReviewTab runId={selectedRun.run_id} review={review} onReview={setReview} />
 
40
  import ReviewTab from '../components/megaeval/ReviewTab';
41
  import ShiftTab from '../components/megaeval/ShiftTab';
42
  import LineageCard from '../components/megaeval/LineageCard';
43
+ import ContainerQualityPanel from '../components/platform/ContainerQualityPanel';
44
+ import ModelComparePanel from '../components/platform/ModelComparePanel';
45
+ import GateStatusPanel from '../components/platform/GateStatusPanel';
46
 
47
  const ACTIVE_STATUSES: RunStatus[] = ['created', 'queued', 'running', 'reducing', 'materializing'];
48
 
 
52
  | 'containers'
53
  | 'investigation'
54
  | 'compare'
55
+ | 'gates'
56
  | 'review'
57
  | 'shift'
58
  | 'lineage';
 
294
  <Tab label="Containers" value="containers" />
295
  <Tab label="Investigation" value="investigation" />
296
  <Tab label="Compare" value="compare" />
297
+ <Tab label="Gates" value="gates" />
298
  <Tab label="Review" value="review" />
299
  <Tab label="Shift" value="shift" />
300
  <Tab label="Lineage" value="lineage" />
301
  </Tabs>
302
 
303
  {tab === 'quality' ? (
304
+ <Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
305
+ <QualityTab runId={selectedRun.run_id} runs={runs} refreshKey={publishTick} />
306
+ <ContainerQualityPanel runId={selectedRun.run_id} refreshKey={publishTick} />
307
+ </Box>
308
  ) : null}
309
  {tab === 'cohorts' ? <CohortTab runId={selectedRun.run_id} refreshKey={publishTick} /> : null}
310
  {tab === 'containers' ? (
311
+ <Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
312
+ <ContainerQualityPanel runId={selectedRun.run_id} refreshKey={publishTick} />
313
+ <ContainersTab
314
+ runId={selectedRun.run_id}
315
+ refreshKey={publishTick}
316
+ drillContainerId={drillContainerId}
317
+ onDrill={setDrillContainerId}
318
+ />
319
+ </Box>
320
  ) : null}
321
  {tab === 'investigation' ? (
322
  <InvestigationTab runId={selectedRun.run_id} onOpenContainer={openContainer} />
323
  ) : null}
324
  {tab === 'compare' ? (
325
+ <Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
326
+ <ModelComparePanel
327
+ runId={selectedRun.run_id}
328
+ baselineRunId={baselineRunId}
329
+ refreshKey={publishTick}
330
+ />
331
+ <CompareTab runId={selectedRun.run_id} baselineRunId={baselineRunId} refreshKey={publishTick} />
332
+ </Box>
333
+ ) : null}
334
+ {tab === 'gates' ? (
335
+ <GateStatusPanel
336
+ candidateRunId={selectedRun.run_id}
337
+ baselineRunId={baselineRunId}
338
+ refreshKey={publishTick}
339
+ />
340
  ) : null}
341
  {tab === 'review' ? (
342
  <ReviewTab runId={selectedRun.run_id} review={review} onReview={setReview} />
src/pages/EvaluationPage.tsx CHANGED
@@ -20,6 +20,7 @@ import {
20
  import { useLabelEval } from '../context/LabelEvalContext';
21
  import EvidencePanel from '../components/labeleval/EvidencePanel';
22
  import CopilotDrawer from '../components/labeleval/CopilotDrawer';
 
23
  import { SectionCard, StatusChip, LoadingBox, ErrorNote, fmtNum, fmtPct } from '../components/labeleval/shared';
24
 
25
  const PAGE_SIZE = 15;
@@ -90,6 +91,7 @@ export default function EvaluationPage() {
90
 
91
  return (
92
  <Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
 
93
  <SectionCard title="Evaluation Record Explorer">
94
  <Box sx={{ display: 'flex', alignItems: 'center', gap: 2, mb: 2 }}>
95
  <Typography variant="body2" sx={{ color: '#8a949e' }}>
 
20
  import { useLabelEval } from '../context/LabelEvalContext';
21
  import EvidencePanel from '../components/labeleval/EvidencePanel';
22
  import CopilotDrawer from '../components/labeleval/CopilotDrawer';
23
+ import GateStatusPanel from '../components/platform/GateStatusPanel';
24
  import { SectionCard, StatusChip, LoadingBox, ErrorNote, fmtNum, fmtPct } from '../components/labeleval/shared';
25
 
26
  const PAGE_SIZE = 15;
 
91
 
92
  return (
93
  <Box sx={{ display: 'flex', flexDirection: 'column', gap: 2 }}>
94
+ <GateStatusPanel candidateRunId={null} baselineRunId={null} />
95
  <SectionCard title="Evaluation Record Explorer">
96
  <Box sx={{ display: 'flex', alignItems: 'center', gap: 2, mb: 2 }}>
97
  <Typography variant="body2" sx={{ color: '#8a949e' }}>
src/pages/QualityEnginePage.tsx CHANGED
@@ -5,6 +5,7 @@ import LabelEvaluationEngine from '../components/labeleval/LabelEvaluationEngine
5
  import StrictQualityValidation from '../components/labeleval/StrictQualityValidation';
6
  import GroundTruthComparison from '../components/labeleval/GroundTruthComparison';
7
  import GraderDisagreement from '../components/labeleval/GraderDisagreement';
 
8
  import { LoadingBox, ErrorNote } from '../components/labeleval/shared';
9
 
10
  export default function QualityEnginePage() {
@@ -20,6 +21,7 @@ export default function QualityEnginePage() {
20
  {metrics.error && !metrics.data ? <ErrorNote error={metrics.error} /> : null}
21
 
22
  <LabelEvaluationEngine pipeline={pipeline ?? null} />
 
23
  <StrictQualityValidation metrics={metrics.data} />
24
  <GroundTruthComparison metrics={metrics.data} />
25
  <GraderDisagreement metrics={metrics.data} />
 
5
  import StrictQualityValidation from '../components/labeleval/StrictQualityValidation';
6
  import GroundTruthComparison from '../components/labeleval/GroundTruthComparison';
7
  import GraderDisagreement from '../components/labeleval/GraderDisagreement';
8
+ import GateStatusPanel from '../components/platform/GateStatusPanel';
9
  import { LoadingBox, ErrorNote } from '../components/labeleval/shared';
10
 
11
  export default function QualityEnginePage() {
 
21
  {metrics.error && !metrics.data ? <ErrorNote error={metrics.error} /> : null}
22
 
23
  <LabelEvaluationEngine pipeline={pipeline ?? null} />
24
+ <GateStatusPanel candidateRunId={null} baselineRunId={null} />
25
  <StrictQualityValidation metrics={metrics.data} />
26
  <GroundTruthComparison metrics={metrics.data} />
27
  <GraderDisagreement metrics={metrics.data} />
src/services/platform.ts ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Platform Phase 1 API client — container quality, multi-model compare, gates, evidence.
3
+ */
4
+ import { usePoll } from './labeleval';
5
+
6
+ async function handle<T>(res: Response): Promise<T> {
7
+ if (!res.ok) {
8
+ const body = await res.text().catch(() => '');
9
+ throw new Error(`API ${res.status}: ${body || res.statusText}`);
10
+ }
11
+ return res.json() as Promise<T>;
12
+ }
13
+
14
+ function get<T>(url: string): Promise<T> {
15
+ return fetch(url).then((res) => handle<T>(res));
16
+ }
17
+
18
+ function post<T>(url: string, body?: unknown): Promise<T> {
19
+ return fetch(url, {
20
+ method: 'POST',
21
+ headers: { 'Content-Type': 'application/json' },
22
+ body: body === undefined ? undefined : JSON.stringify(body),
23
+ }).then((res) => handle<T>(res));
24
+ }
25
+
26
+ export type VerificationRates = {
27
+ n_objects: number;
28
+ verified: number;
29
+ unverified: number;
30
+ disputed: number;
31
+ auto_accepted: number;
32
+ hitl: number;
33
+ reviewed: number;
34
+ verified_rate: number | null;
35
+ unverified_rate: number | null;
36
+ disputed_rate: number | null;
37
+ auto_accept_rate: number | null;
38
+ hitl_rate: number | null;
39
+ review_coverage: number | null;
40
+ };
41
+
42
+ export type ContainerQualityRow = {
43
+ container_id: number;
44
+ status: string | null;
45
+ risk_score: number;
46
+ precision: number | null;
47
+ recall: number | null;
48
+ f1: number | null;
49
+ mean_iou: number | null;
50
+ verification: VerificationRates;
51
+ dims: Record<string, string>;
52
+ };
53
+
54
+ export type ContainerQualityProfile = {
55
+ run_id: string;
56
+ model_version: string;
57
+ total: number;
58
+ summary: {
59
+ precision: number | null;
60
+ recall: number | null;
61
+ f1: number | null;
62
+ mean_iou: number | null;
63
+ verification: VerificationRates;
64
+ };
65
+ containers: ContainerQualityRow[];
66
+ verification_headline: VerificationRates;
67
+ };
68
+
69
+ export type GateResultRow = {
70
+ result_id: string;
71
+ gate_id: string;
72
+ gate_type: string;
73
+ passed: boolean | null;
74
+ ready: boolean;
75
+ message: string;
76
+ failures: Array<Record<string, unknown>>;
77
+ thresholds: Record<string, number>;
78
+ };
79
+
80
+ export type GateStatusResponse = {
81
+ gates: GateResultRow[];
82
+ summary: {
83
+ wired_count: number;
84
+ unwired_count: number;
85
+ wired_all_passed: boolean;
86
+ release_ready: boolean;
87
+ unwired_gate_types: string[];
88
+ };
89
+ };
90
+
91
+ export type ModelCompareResponse = {
92
+ baseline_run_id: string;
93
+ run_ids: string[];
94
+ models: Array<{
95
+ run_id: string;
96
+ model_version: string;
97
+ role: string;
98
+ headline: Record<string, number | null>;
99
+ }>;
100
+ metric_matrix: Array<{
101
+ metric: string;
102
+ deltas_vs_baseline: Record<string, number | null>;
103
+ [model: string]: unknown;
104
+ }>;
105
+ pairwise: Array<{
106
+ candidate_model: string;
107
+ recommendation: string;
108
+ blockers: string[];
109
+ }>;
110
+ recommendations: Array<{ candidate: string; recommendation: string; blocker_count: number }>;
111
+ };
112
+
113
+ export const getContainerQuality = (runId: string, limit = 25) =>
114
+ get<ContainerQualityProfile>(
115
+ `/api/containers/quality?run_id=${encodeURIComponent(runId)}&limit=${limit}`
116
+ );
117
+
118
+ export const compareModels = (runIds: string[], baselineRunId?: string | null) =>
119
+ post<ModelCompareResponse>('/api/models/compare', {
120
+ run_ids: runIds,
121
+ baseline_run_id: baselineRunId ?? undefined,
122
+ });
123
+
124
+ export const getGateStatus = (params?: {
125
+ sequence_id?: string;
126
+ candidate_run_id?: string;
127
+ baseline_run_id?: string;
128
+ }) => {
129
+ const q = new URLSearchParams();
130
+ if (params?.sequence_id) q.set('sequence_id', params.sequence_id);
131
+ if (params?.candidate_run_id) q.set('candidate_run_id', params.candidate_run_id);
132
+ if (params?.baseline_run_id) q.set('baseline_run_id', params.baseline_run_id);
133
+ const qs = q.toString();
134
+ return get<GateStatusResponse>(`/api/gates/status${qs ? `?${qs}` : ''}`);
135
+ };
136
+
137
+ export const getEvaluationLevels = () =>
138
+ get<{ levels: string[]; backends: Record<string, string> }>('/api/evaluations/levels');
139
+
140
+ export const exportEvidence = (body: Record<string, unknown>) =>
141
+ post<{ package: Record<string, unknown>; path: string | null }>('/api/evaluations/evidence', body);
142
+
143
+ export function useContainerQuality(runId: string | null, refreshKey = 0) {
144
+ return usePoll(
145
+ () => (runId ? getContainerQuality(runId) : Promise.resolve(null)),
146
+ null,
147
+ [runId, refreshKey]
148
+ );
149
+ }
150
+
151
+ export function useGateStatus(
152
+ candidateRunId: string | null,
153
+ baselineRunId: string | null,
154
+ refreshKey = 0
155
+ ) {
156
+ return usePoll(
157
+ () =>
158
+ getGateStatus({
159
+ candidate_run_id: candidateRunId ?? undefined,
160
+ baseline_run_id: baselineRunId ?? undefined,
161
+ }),
162
+ 8000,
163
+ [candidateRunId, baselineRunId, refreshKey]
164
+ );
165
+ }
tests/test_platform/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ # Phase 1 platform foundation tests
tests/test_platform/test_foundation.py ADDED
@@ -0,0 +1,285 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Phase 1 platform foundation tests — metrics, container quality, compare, gates, evidence."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+
8
+ import pytest
9
+
10
+ from sensorflow.platform.metrics_engine import (
11
+ aggregate_container_rows,
12
+ container_quality_metrics,
13
+ delta_metrics,
14
+ precision_recall_f1,
15
+ verification_rates,
16
+ )
17
+ from sensorflow.platform.container_quality import profile_from_counts
18
+ from sensorflow.platform.levels import AggregateLevel, EvaluationScope, summarize_scope
19
+ from sensorflow.platform.gates import (
20
+ DEFAULT_GATE_CONFIG,
21
+ evaluate_multi_gates,
22
+ list_gate_defs,
23
+ load_gate_config,
24
+ )
25
+ from sensorflow.platform.evidence import build_evidence_package, export_evidence_package
26
+ from sensorflow.platform.compare import compare_models
27
+ from sensorflow.platform.entities import EvidencePackage, Provenance
28
+
29
+
30
+ # ------------------------------------------------------------------ metrics
31
+
32
+
33
+ def test_precision_recall_f1_basic():
34
+ out = precision_recall_f1(tp=8, fp=2, fn=2)
35
+ assert out["precision"] == 0.8
36
+ assert out["recall"] == 0.8
37
+ assert out["f1"] == 0.8
38
+
39
+
40
+ def test_precision_recall_empty():
41
+ out = precision_recall_f1(0, 0, 0)
42
+ assert out["precision"] is None
43
+ assert out["recall"] is None
44
+ assert out["f1"] is None
45
+
46
+
47
+ def test_verification_rates():
48
+ v = verification_rates(
49
+ n_objects=100,
50
+ verified=40,
51
+ reviewed=55,
52
+ disputed=5,
53
+ auto_accepted=30,
54
+ hitl=15,
55
+ )
56
+ assert v["verified_rate"] == 0.4
57
+ assert v["hitl_rate"] == 0.15
58
+ assert v["disputed_rate"] == 0.05
59
+ assert v["auto_accept_rate"] == 0.3
60
+ assert v["unverified"] == 60
61
+
62
+
63
+ def test_container_quality_metrics_bundle():
64
+ m = container_quality_metrics(
65
+ tp=10, fp=5, fn=5, sum_iou=7.0, n_objects=20,
66
+ verified=8, reviewed=12, hitl=4, anomalies=2,
67
+ )
68
+ assert m["precision"] == pytest.approx(10 / 15, rel=1e-3)
69
+ assert m["mean_iou"] == 0.7
70
+ assert m["verification"]["verified_rate"] == 0.4
71
+ assert m["anomaly_rate"] == 0.1
72
+
73
+
74
+ def test_aggregate_container_rows():
75
+ rows = [
76
+ {"tp": 5, "fp": 1, "fn": 1, "sum_iou": 4.0, "n_objects": 10, "verified": 3, "reviewed": 4},
77
+ {"tp": 5, "fp": 1, "fn": 1, "sum_iou": 3.5, "n_objects": 10, "verified": 5, "reviewed": 6},
78
+ ]
79
+ agg = aggregate_container_rows(rows)
80
+ assert agg["tp"] == 10
81
+ assert agg["verification"]["verified"] == 8
82
+
83
+
84
+ def test_delta_metrics():
85
+ deltas = delta_metrics(
86
+ {"precision": 0.9, "recall": 0.8},
87
+ {"precision": 0.88, "recall": 0.85},
88
+ keys=["precision", "recall"],
89
+ )
90
+ assert deltas[0]["delta"] == pytest.approx(-0.02)
91
+ assert deltas[1]["delta"] == pytest.approx(0.05)
92
+
93
+
94
+ def test_profile_from_counts():
95
+ p = profile_from_counts(
96
+ container_id="c-1", tp=4, fp=1, fn=1, sum_iou=3.2,
97
+ n_objects=10, verified=2, reviewed=5, hitl=3,
98
+ )
99
+ assert p["container_id"] == "c-1"
100
+ assert p["f1"] is not None
101
+ assert p["verification"]["hitl"] == 3
102
+
103
+
104
+ # ------------------------------------------------------------------ levels
105
+
106
+
107
+ def test_aggregate_ladder():
108
+ ladder = AggregateLevel.ladder()
109
+ assert ladder[0] == AggregateLevel.FRAME
110
+ assert ladder[-1] == AggregateLevel.POPULATION
111
+ assert AggregateLevel.CONTAINER.parent() == AggregateLevel.DATASET
112
+ assert AggregateLevel.FRAME.child() is None
113
+
114
+
115
+ def test_evaluation_scope_summary():
116
+ scope = EvaluationScope(
117
+ level=AggregateLevel.CONTAINER,
118
+ run_id="run-1",
119
+ mega_container_id=42,
120
+ model_version="model-v1",
121
+ )
122
+ s = summarize_scope(scope)
123
+ assert s["backend"] == "megaeval"
124
+ assert s["refs"]["mega_container_id"] == 42
125
+
126
+
127
+ # ------------------------------------------------------------------ gates
128
+
129
+
130
+ def test_gate_defs_include_multi_gate_skeleton(tmp_path: Path):
131
+ cfg_path = tmp_path / "gates.json"
132
+ cfg = load_gate_config(cfg_path)
133
+ assert cfg_path.exists()
134
+ defs = list_gate_defs(cfg)
135
+ types = {d.gate_type for d in defs}
136
+ for required in ("scenario", "coverage", "regression", "safety", "release", "quality", "launch"):
137
+ assert required in types
138
+
139
+
140
+ def test_evaluate_multi_gates_skeleton(tmp_path: Path, monkeypatch):
141
+ cfg_path = tmp_path / "gates.json"
142
+ monkeypatch.setattr(
143
+ "sensorflow.platform.gates.DEFAULT_GATE_CONFIG_PATH", cfg_path
144
+ )
145
+ # rewrite load to use tmp
146
+ from sensorflow.platform import gates as gates_mod
147
+ monkeypatch.setattr(gates_mod, "DEFAULT_GATE_CONFIG_PATH", cfg_path)
148
+ out = gates_mod.evaluate_multi_gates()
149
+ assert out["summary"]["unwired_count"] >= 3
150
+ assert out["summary"]["release_ready"] is False
151
+ types = [g["gate_type"] for g in out["gates"]]
152
+ assert "scenario" in types and "safety" in types
153
+ skeleton = [g for g in out["gates"] if g["gate_type"] == "safety"][0]
154
+ assert skeleton["ready"] is False
155
+ assert skeleton["passed"] is None
156
+
157
+
158
+ def test_gate_config_thresholds_are_config_driven():
159
+ assert "map_3d" in DEFAULT_GATE_CONFIG["gates"]["quality"]["thresholds"]
160
+ assert DEFAULT_GATE_CONFIG["gates"]["safety"]["ready"] is False
161
+
162
+
163
+ # ------------------------------------------------------------------ evidence
164
+
165
+
166
+ def test_evidence_package_stub(tmp_path: Path, monkeypatch):
167
+ monkeypatch.setattr(
168
+ "sensorflow.platform.evidence.EVIDENCE_DIR", tmp_path
169
+ )
170
+ pkg = build_evidence_package(evaluation_id="eval-test")
171
+ assert isinstance(pkg, EvidencePackage)
172
+ assert pkg.evaluation_id == "eval-test"
173
+ assert "model_versions" in pkg.placeholders or pkg.model_versions
174
+ path = export_evidence_package(pkg, tmp_path / f"{pkg.package_id}.json")
175
+ assert path.exists()
176
+ raw = json.loads(path.read_text())
177
+ assert raw["evaluation_id"] == "eval-test"
178
+ assert "gate_results" in raw
179
+ assert "compute_usage" in raw
180
+ assert "provenance" in raw
181
+
182
+ # append-only: second export with same path gets a revision file
183
+ path2 = export_evidence_package(pkg, path)
184
+ assert path2 != path
185
+ assert path2.exists()
186
+
187
+
188
+ # ------------------------------------------------------------------ compare (unit, no mega store)
189
+
190
+
191
+ def test_compare_models_requires_two_runs():
192
+ with pytest.raises(ValueError):
193
+ compare_models(["only-one"])
194
+
195
+
196
+ class _FakeRun:
197
+ def __init__(self, run_id, model_version, headline, status="published"):
198
+ self.run_id = run_id
199
+ self.model_version = model_version
200
+ self.headline = headline
201
+ self.status = status
202
+ self.population_id = "pop-1"
203
+ self.label_version = "labels-v1"
204
+
205
+
206
+ class _FakeStore:
207
+ def __init__(self, runs):
208
+ self.runs = {r.run_id: r for r in runs}
209
+
210
+ def artifacts(self, run_id):
211
+ raise NotImplementedError
212
+
213
+
214
+ def test_compare_models_matrix(monkeypatch):
215
+ a = _FakeRun("run-a", "model-A", {
216
+ "precision": 0.9, "recall": 0.8, "f1": 0.85,
217
+ "mean_iou": 0.7, "safety_recall": 0.75, "anomaly_rate": 0.1,
218
+ })
219
+ b = _FakeRun("run-b", "model-B", {
220
+ "precision": 0.88, "recall": 0.82, "f1": 0.85,
221
+ "mean_iou": 0.71, "safety_recall": 0.74, "anomaly_rate": 0.09,
222
+ })
223
+ c = _FakeRun("run-c", "model-C", {
224
+ "precision": 0.91, "recall": 0.79, "f1": 0.845,
225
+ "mean_iou": 0.69, "safety_recall": 0.78, "anomaly_rate": 0.11,
226
+ })
227
+ store = _FakeStore([a, b, c])
228
+
229
+ def fake_compare(store, cand, base, policy=None):
230
+ return {
231
+ "recommendation": "PROMOTE",
232
+ "blockers": [],
233
+ "headline_deltas": [],
234
+ "per_class": [],
235
+ "regressions": [],
236
+ }
237
+
238
+ monkeypatch.setattr("sensorflow.megaeval.analysis.compare_runs", fake_compare)
239
+ out = compare_models(
240
+ ["run-a", "run-b", "run-c"],
241
+ baseline_run_id="run-a",
242
+ store=store,
243
+ )
244
+ assert len(out["models"]) == 3
245
+ assert len(out["pairwise"]) == 2
246
+ assert len(out["metric_matrix"]) >= 3
247
+ recall_row = next(r for r in out["metric_matrix"] if r["metric"] == "recall")
248
+ assert recall_row["model-B"] == 0.82
249
+ assert recall_row["deltas_vs_baseline"]["model-B"] == pytest.approx(0.02)
250
+
251
+
252
+ # ------------------------------------------------------------------ API smoke (optional FastAPI)
253
+
254
+
255
+ def test_platform_router_mounts():
256
+ from fastapi.testclient import TestClient
257
+ from sensorflow.platform.api import router
258
+ from fastapi import FastAPI
259
+
260
+ app = FastAPI()
261
+ app.include_router(router)
262
+ client = TestClient(app)
263
+ levels = client.get("/api/evaluations/levels")
264
+ assert levels.status_code == 200
265
+ body = levels.json()
266
+ assert "population" in body["levels"]
267
+ assert body["backends"]["container"] == "megaeval"
268
+
269
+ gates = client.get("/api/gates")
270
+ assert gates.status_code == 200
271
+ assert any(g["gate_type"] == "scenario" for g in gates.json()["gates"])
272
+
273
+ status = client.get("/api/gates/status")
274
+ assert status.status_code == 200
275
+ assert status.json()["summary"]["release_ready"] is False
276
+
277
+ evidence = client.post("/api/evaluations/evidence", json={
278
+ "evaluation_id": "api-eval-1",
279
+ "persist": False,
280
+ })
281
+ assert evidence.status_code == 200
282
+ assert evidence.json()["package"]["evaluation_id"] == "api-eval-1"
283
+
284
+ bad = client.post("/api/models/compare", json={"run_ids": ["x"]})
285
+ assert bad.status_code == 422 or bad.status_code == 400