app_backend.py

#!/usr/bin/env python3
import os
import sys
import json
import subprocess
import signal
from pathlib import Path
from typing import List, Optional, Any, Dict
from pydantic import BaseModel
import httpx
from fastapi import FastAPI, HTTPException, Query
from fastapi.staticfiles import StaticFiles
from fastapi.responses import FileResponse, PlainTextResponse

app = FastAPI(title="Sensorflow Studio Backend")

# State tracking
ACTIVE_TRAIN_PROC: Optional[subprocess.Popen] = None
ACTIVE_TRAIN_EXECUTION_ID: Optional[str] = None
TRAINING_LOGS_ACCUMULATED = ""
TRAINING_LOSSES: List[float] = []
TOTAL_EPOCHS = 10
CURRENT_EPOCH = 0
LAST_TRAIN_EXIT_CODE: Optional[int] = None
LAST_TRAIN_COMMAND: Optional[List[str]] = None

# Configuration
CONFIG_PATH = Path("runs/studio_config.json")
CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)

class StudioConfig(BaseModel):
    yaml_path: str = "coco8.yaml"
    source_path: str = "data"
    dataset_type: str = "local"
    model_type: str = "yolov8"
    pipeline_mode: str = "3d"
    sam_checkpoint: str = "models/sam_vit_b.pth"
    vendors: List[str] = ["alpamayo"]
    gate_thresholds_path: str = "runs/pipeline/gate_thresholds.json"
    sequence_id: str = "seq_001"
    waymo_root: Optional[str] = None
    alpamayo_root: Optional[str] = None
    a2d2_root: Optional[str] = None
    allow_stub: bool = True

class TrainParams(BaseModel):
    model: str = "yolov8n.pt"
    epochs: int = 10
    batch: int = 16
    device: str = "cpu"
    data: str = "coco8.yaml"

class InferParams(BaseModel):
    weights: str = "runs/detect/coco_finetuned/weights/best.pt"
    conf: float = 0.25
    iou: float = 0.45
    source: str = "data"

class ExportParams(BaseModel):
    weights: str = "runs/detect/coco_finetuned/weights/best.pt"
    format: str = "onnx"

def load_config() -> StudioConfig:
    if CONFIG_PATH.exists():
        try:
            with open(CONFIG_PATH, "r") as f:
                data = json.load(f)
                return StudioConfig(**data)
        except Exception:
            pass
    return StudioConfig()


def resolve_active_sequence_id(fallback: str = "seq_001") -> str:
    """Prefer studio config, then pipeline state from load-all / ingest."""
    config = load_config()
    if config.sequence_id:
        return config.sequence_id
    state_path = Path("runs/pipeline/state.json")
    if state_path.exists():
        try:
            state = json.loads(state_path.read_text())
            active = state.get("_active_sequence")
            if active:
                return str(active)
        except Exception:
            pass
    return fallback


SAM_CHECKPOINT_HINT = (
    "Download SAM ViT-B: "
    "mkdir -p models && "
    "curl -L -o models/sam_vit_b.pth "
    "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth"
)

@app.post("/api/config")
def save_config(cfg: StudioConfig):
    with open(CONFIG_PATH, "w") as f:
        json.dump(cfg.dict(), f, indent=2)
    return {"status": "ok", "config": cfg}

@app.get("/api/config")
def get_config():
    return load_config()

class SaveYamlRequest(BaseModel):
    path: str
    content: str

@app.get("/api/yaml/content")
def get_yaml_content(path: str = "coco8.yaml"):
    try:
        resolved_path = Path(path)
        if not resolved_path.is_absolute():
            resolved_path = Path(__file__).parent / resolved_path
        if not resolved_path.exists():
            return {"status": "error", "message": f"File {path} does not exist."}
        with open(resolved_path, "r") as f:
            content = f.read()
        return {"status": "ok", "path": str(resolved_path), "content": content}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to read YAML file: {str(e)}")

@app.post("/api/yaml/save")
def save_yaml_content(req: SaveYamlRequest):
    try:
        resolved_path = Path(req.path)
        if not resolved_path.is_absolute():
            resolved_path = Path(__file__).parent / resolved_path
        with open(resolved_path, "w") as f:
            f.write(req.content)
        return {"status": "ok", "message": f"YAML config {req.path} saved successfully."}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to write YAML file: {str(e)}")

@app.get("/api/precheck")
def precheck():
    """Script verification — existence alone is NOT success."""
    from sensorflow.execution_ops import verify_script

    reports = {
        "train.py": verify_script("train.py"),
        "infer.py": verify_script("infer.py"),
        "autograder.py": verify_script("autograder.py"),
    }
    all_ok = all(
        r.get("exists") and r.get("syntax_valid") and r.get("dry_run_ok")
        for r in reports.values()
    )
    any_missing = any(not r.get("exists") for r in reports.values())
    return {
        "status": "ok" if all_ok else ("warning" if not any_missing else "failed"),
        "message": (
            "Scripts exist, syntax-valid, and --help dry-run succeeded."
            if all_ok
            else "Script verification incomplete — see reports (file Found alone is not enough)."
        ),
        "scripts": reports,
        "verified": all_ok,
    }


@app.get("/api/health")
def health():
    from sensorflow.execution_ops import backend_health
    from sensorflow.execution_ledger import get_strict_mode

    payload = backend_health()
    payload["strict_mode"] = get_strict_mode()
    payload["backend_connected"] = True
    return payload


@app.get("/api/strict-mode")
def get_strict_mode_api():
    from sensorflow.execution_ledger import get_strict_mode

    return {"enabled": get_strict_mode()}


@app.post("/api/strict-mode")
def set_strict_mode_api(params: dict):
    from sensorflow.execution_ledger import set_strict_mode

    enabled = bool(params.get("enabled", False))
    return set_strict_mode(enabled)


@app.get("/api/executions")
def list_executions_api(
    operation: Optional[str] = None,
    status: Optional[str] = None,
    limit: int = Query(50, ge=1, le=500),
):
    from sensorflow.execution_ledger import list_executions

    return {"status": "ok", "executions": list_executions(operation=operation, status=status, limit=limit)}


@app.get("/api/executions/{execution_id}")
def get_execution_api(execution_id: str):
    from sensorflow.execution_ledger import load_execution

    try:
        return load_execution(execution_id)
    except FileNotFoundError:
        raise HTTPException(status_code=404, detail=f"Execution not found: {execution_id}")


@app.get("/api/executions/{execution_id}/log")
def get_execution_log_api(execution_id: str):
    from sensorflow.execution_ledger import get_log_text, load_execution

    try:
        load_execution(execution_id)
    except FileNotFoundError:
        raise HTTPException(status_code=404, detail=f"Execution not found: {execution_id}")
    return PlainTextResponse(get_log_text(execution_id) or "(no log lines)")


@app.post("/api/yaml/validate")
def validate_yaml_api(params: dict):
    from sensorflow.execution_ops import validate_yaml_semantics
    from sensorflow.execution_ledger import create_execution, mark_running, finalize

    path = params.get("path") or params.get("yaml_path") or "coco8.yaml"
    record = create_execution(
        "yaml_validate",
        configuration_snapshot={"yaml_path": path},
        input_artifacts=[path],
    )
    mark_running(record["execution_id"])
    report = validate_yaml_semantics(path)
    final = finalize(
        record["execution_id"],
        report.get("status") or "FAILED",
        metrics={
            "class_count": report.get("class_count"),
            "image_counts": report.get("image_counts"),
            "label_counts": report.get("label_counts"),
        },
        errors=report.get("errors"),
        warnings=report.get("warnings"),
        output_artifacts=[{"path": path, "hash": report.get("content_hash")}],
        process_invoked=False,
        outputs_valid=report.get("status") == "SUCCEEDED",
    )
    report["execution_id"] = final["execution_id"]
    report["execution"] = {
        "execution_id": final["execution_id"],
        "status": final["status"],
        "verified": final.get("verified"),
        "duration_ms": final.get("duration_ms"),
    }
    return report


@app.get("/api/scripts/verify")
def scripts_verify_api():
    from sensorflow.execution_ops import verify_script

    return {
        "status": "ok",
        "scripts": {
            "train.py": verify_script("train.py"),
            "infer.py": verify_script("infer.py"),
            "autograder.py": verify_script("autograder.py"),
        },
    }


@app.post("/api/dataset/load")
def dataset_load(params: dict):
    """Real Load & Preprocess with discovery evidence (not catalog KPIs)."""
    from sensorflow.execution_ops import load_and_preprocess
    from sensorflow.execution_ledger import create_execution, mark_running, finalize

    source_path = params.get("source_path") or load_config().source_path
    yaml_path = params.get("yaml_path") or load_config().yaml_path
    dataset_type = params.get("dataset_type") or load_config().dataset_type

    record = create_execution(
        "dataset_load",
        configuration_snapshot={
            "source_path": source_path,
            "yaml_path": yaml_path,
            "dataset_type": dataset_type,
        },
        input_artifacts=[source_path, yaml_path],
    )
    mark_running(record["execution_id"])
    result = load_and_preprocess(source_path, yaml_path=yaml_path, dataset_type=dataset_type)
    disc = result["discovery"]
    final = finalize(
        record["execution_id"],
        result["status"],
        records_discovered=disc.get("images_discovered", 0),
        records_processed=disc.get("images_discovered", 0),
        records_succeeded=disc.get("images_readable", 0),
        records_failed=disc.get("images_corrupt", 0),
        metrics=result["metrics"],
        errors=disc.get("errors"),
        warnings=disc.get("warnings"),
        output_artifacts=[{"path": result["manifest_path"], "kind": "load_manifest"}],
        process_invoked=False,
        outputs_valid=disc.get("images_readable", 0) > 0 and result["status"] in (
            "SUCCEEDED",
            "PARTIAL_SUCCESS",
        ),
    )
    return {
        "status": final["status"],
        "execution_id": final["execution_id"],
        "verified": final.get("verified"),
        "duration_ms": final.get("duration_ms"),
        "message": (
            f"Loaded {disc.get('images_readable', 0)}/{disc.get('images_discovered', 0)} images "
            f"from {source_path}"
            if disc.get("images_readable")
            else f"FAILED: 0 images loaded from {source_path}"
        ),
        "discovery": disc,
        "metrics": result["metrics"],
        "yaml_validation": result.get("yaml_validation"),
        "reconciliation": result.get("reconciliation"),
        "manifest_path": result.get("manifest_path"),
        "catalog_only": False,
        "browsable": bool(result["metrics"].get("browsable")),
        "events": final.get("events"),
    }


@app.post("/api/train/start")
def start_train(params: TrainParams):
    global ACTIVE_TRAIN_PROC, ACTIVE_TRAIN_EXECUTION_ID, TRAINING_LOGS_ACCUMULATED
    global TRAINING_LOSSES, TOTAL_EPOCHS, CURRENT_EPOCH, LAST_TRAIN_EXIT_CODE, LAST_TRAIN_COMMAND
    from sensorflow.execution_ledger import create_execution, mark_running, finalize

    if ACTIVE_TRAIN_PROC and ACTIVE_TRAIN_PROC.poll() is None:
        raise HTTPException(status_code=400, detail="Training is already running.")

    weights_path = Path(params.model)
    # Allow ultralytics named weights (e.g. yolov8n.pt) that may download; still record path
    yaml_path = Path(params.data)
    if not yaml_path.exists():
        record = create_execution(
            "training",
            configuration_snapshot=params.dict(),
            input_artifacts=[params.data, params.model],
        )
        finalize(
            record["execution_id"],
            "FAILED",
            errors=[f"Dataset YAML not found: {params.data}"],
            process_invoked=False,
            outputs_valid=False,
        )
        raise HTTPException(
            status_code=400,
            detail={"message": f"Dataset YAML not found: {params.data}", "execution_id": record["execution_id"]},
        )

    TRAINING_LOGS_ACCUMULATED = f"Starting YOLO training with base model {params.model}...\n"
    TRAINING_LOSSES = []
    TOTAL_EPOCHS = params.epochs
    CURRENT_EPOCH = 0
    LAST_TRAIN_EXIT_CODE = None

    cmd = [
        sys.executable, "train.py",
        "--epochs", str(params.epochs),
        "--batch", str(params.batch),
        "--device", params.device,
        "--model", params.model,
        "--data", params.data,
    ]
    LAST_TRAIN_COMMAND = cmd

    record = create_execution(
        "training",
        configuration_snapshot=params.dict(),
        input_artifacts=[params.data, params.model],
        command=cmd,
    )
    ACTIVE_TRAIN_EXECUTION_ID = record["execution_id"]

    try:
        ACTIVE_TRAIN_PROC = subprocess.Popen(
            cmd,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            text=True,
            bufsize=1,
            preexec_fn=os.setsid if os.name != "nt" else None,
        )
    except Exception as e:
        finalize(
            record["execution_id"],
            "FAILED",
            errors=[f"Failed to spawn training process: {e}"],
            process_invoked=False,
            outputs_valid=False,
        )
        ACTIVE_TRAIN_EXECUTION_ID = None
        raise HTTPException(status_code=500, detail=f"Failed to spawn training process: {str(e)}")

    mark_running(record["execution_id"], process_id=ACTIVE_TRAIN_PROC.pid)
    return {
        "status": "started",
        "execution_id": record["execution_id"],
        "process_id": ACTIVE_TRAIN_PROC.pid,
        "command": cmd,
    }


@app.post("/api/train/stop")
def stop_train():
    global ACTIVE_TRAIN_PROC, ACTIVE_TRAIN_EXECUTION_ID, LAST_TRAIN_EXIT_CODE
    from sensorflow.execution_ledger import finalize, append_log

    if not ACTIVE_TRAIN_PROC or ACTIVE_TRAIN_PROC.poll() is not None:
        return {"status": "not_running", "execution_id": ACTIVE_TRAIN_EXECUTION_ID}

    exec_id = ACTIVE_TRAIN_EXECUTION_ID
    try:
        if os.name != "nt":
            os.killpg(os.getpgid(ACTIVE_TRAIN_PROC.pid), signal.SIGTERM)
        else:
            ACTIVE_TRAIN_PROC.terminate()
        ACTIVE_TRAIN_PROC.wait(timeout=3)
    except Exception:
        try:
            ACTIVE_TRAIN_PROC.kill()
        except Exception:
            pass

    LAST_TRAIN_EXIT_CODE = ACTIVE_TRAIN_PROC.poll()
    if exec_id:
        append_log(exec_id, "Training terminated by user")
        finalize(
            exec_id,
            "CANCELLED",
            exit_code=LAST_TRAIN_EXIT_CODE,
            process_invoked=True,
            outputs_valid=False,
            errors=["Cancelled by user"],
        )
    ACTIVE_TRAIN_PROC = None
    return {"status": "stopped", "execution_id": exec_id, "exit_code": LAST_TRAIN_EXIT_CODE}


@app.get("/api/train/status")
def get_train_status():
    global ACTIVE_TRAIN_PROC, ACTIVE_TRAIN_EXECUTION_ID, TRAINING_LOGS_ACCUMULATED
    global TRAINING_LOSSES, CURRENT_EPOCH, TOTAL_EPOCHS, LAST_TRAIN_EXIT_CODE, LAST_TRAIN_COMMAND
    from sensorflow.execution_ledger import append_log, finalize, sha256_file
    from sensorflow.execution_ops import artifact_info

    running = False
    just_finished = False
    exit_code = LAST_TRAIN_EXIT_CODE

    if ACTIVE_TRAIN_PROC:
        import select

        if ACTIVE_TRAIN_PROC.poll() is None:
            running = True
            while True:
                r, _, _ = select.select([ACTIVE_TRAIN_PROC.stdout], [], [], 0.02)
                if ACTIVE_TRAIN_PROC.stdout in r:
                    line = ACTIVE_TRAIN_PROC.stdout.readline()
                    if not line:
                        break
                    TRAINING_LOGS_ACCUMULATED += line
                    if ACTIVE_TRAIN_EXECUTION_ID:
                        try:
                            append_log(ACTIVE_TRAIN_EXECUTION_ID, line.rstrip())
                        except Exception:
                            pass
                    if "epoch" in line.lower() or "/" in line:
                        try:
                            parts = line.split()
                            if len(parts) >= 3 and "/" in parts[0]:
                                ep_part = parts[0].split("/")
                                epoch_num = int(ep_part[0])
                                CURRENT_EPOCH = epoch_num
                                for p in parts[2:]:
                                    try:
                                        val = float(p)
                                        if 0.0 < val < 10.0:
                                            if len(TRAINING_LOSSES) < epoch_num:
                                                TRAINING_LOSSES.append(val)
                                            else:
                                                TRAINING_LOSSES[epoch_num - 1] = val
                                            break
                                    except ValueError:
                                        pass
                        except Exception:
                            pass
                else:
                    break
        else:
            remaining = ACTIVE_TRAIN_PROC.stdout.read()
            if remaining:
                TRAINING_LOGS_ACCUMULATED += remaining
            exit_code = ACTIVE_TRAIN_PROC.poll()
            LAST_TRAIN_EXIT_CODE = exit_code
            just_finished = True
            ACTIVE_TRAIN_PROC = None

    progress = 0.0
    if TOTAL_EPOCHS > 0 and CURRENT_EPOCH > 0:
        progress = min(1.0, float(CURRENT_EPOCH) / float(TOTAL_EPOCHS))
    elif not running and exit_code == 0 and CURRENT_EPOCH > 0:
        progress = 1.0
    # Never invent progress=1.0 on failure or empty parse

    checkpoint = Path("runs/detect/coco_finetuned/weights/best.pt")
    ckpt_info = artifact_info(checkpoint) if checkpoint.exists() else {"path": str(checkpoint), "exists": False}

    if just_finished and ACTIVE_TRAIN_EXECUTION_ID:
        status = "SUCCEEDED" if exit_code == 0 else "FAILED"
        if exit_code is None:
            status = "FAILED"
        finalize(
            ACTIVE_TRAIN_EXECUTION_ID,
            status,
            exit_code=exit_code,
            metrics={
                "epochs_requested": TOTAL_EPOCHS,
                "epochs_observed": CURRENT_EPOCH,
                "losses_parsed": len(TRAINING_LOSSES),
                "losses": TRAINING_LOSSES,
                "checkpoint": ckpt_info,
            },
            output_artifacts=[ckpt_info] if ckpt_info.get("exists") else [],
            process_invoked=True,
            outputs_valid=bool(ckpt_info.get("exists")) and exit_code == 0,
            errors=[] if exit_code == 0 else [f"Training exit_code={exit_code}"],
        )

    return {
        "running": running,
        "progress": progress,
        "logs": TRAINING_LOGS_ACCUMULATED[-20000:],
        "losses": TRAINING_LOSSES,  # only real parsed losses — never fabricated
        "losses_fabricated": False,
        "exit_code": exit_code,
        "process_id": None if not running else (ACTIVE_TRAIN_PROC.pid if ACTIVE_TRAIN_PROC else None),
        "command": LAST_TRAIN_COMMAND,
        "execution_id": ACTIVE_TRAIN_EXECUTION_ID,
        "epochs_observed": CURRENT_EPOCH,
        "epochs_requested": TOTAL_EPOCHS,
        "checkpoint": ckpt_info,
        "status": (
            "RUNNING" if running else (
                "SUCCEEDED" if exit_code == 0 else (
                    "FAILED" if exit_code not in (None, 0) else "NOT_EXECUTED"
                )
            )
        ),
    }


@app.post("/api/infer/run")
def run_infer(params: InferParams):
    from sensorflow.execution_ledger import create_execution, mark_running, finalize, append_log
    from sensorflow.execution_ops import discover_images, artifact_info

    weights = Path(params.weights)
    source = Path(params.source)
    out_dir = Path("runs/infer")

    record = create_execution(
        "inference",
        configuration_snapshot=params.dict(),
        input_artifacts=[params.source, params.weights],
        command=[
            sys.executable, "infer.py",
            "--source", params.source,
            "--weights", params.weights,
            "--conf", str(params.conf),
            "--iou", str(params.iou),
            "--output", "runs/infer",
        ],
    )

    if not weights.exists() and not str(params.weights).endswith(".pt"):
        finalize(
            record["execution_id"],
            "FAILED",
            errors=[f"Model checkpoint not found: {params.weights}"],
            process_invoked=False,
            outputs_valid=False,
        )
        raise HTTPException(
            status_code=400,
            detail={"message": f"Model checkpoint not found: {params.weights}", "execution_id": record["execution_id"]},
        )
    if not weights.exists():
        # Named ultralytics weight may auto-download; still warn if missing before run
        pass

    discovery = discover_images(params.source)
    if discovery["images_discovered"] == 0:
        finalize(
            record["execution_id"],
            "FAILED",
            records_discovered=0,
            records_processed=0,
            records_succeeded=0,
            records_failed=0,
            errors=[f"No images found at source: {params.source}"],
            metrics={"images_discovered": 0},
            process_invoked=False,
            outputs_valid=False,
        )
        return {
            "status": "FAILED",
            "execution_id": record["execution_id"],
            "images": [],
            "records_discovered": 0,
            "records_succeeded": 0,
            "records_failed": 0,
            "message": f"No images found at source: {params.source}",
            "model": params.weights,
            "checkpoint": artifact_info(weights),
        }

    if not weights.exists():
        finalize(
            record["execution_id"],
            "FAILED",
            errors=[f"Missing model checkpoint: {params.weights}"],
            records_discovered=discovery["images_discovered"],
            process_invoked=False,
            outputs_valid=False,
        )
        raise HTTPException(
            status_code=400,
            detail={
                "message": f"Missing model checkpoint: {params.weights}",
                "execution_id": record["execution_id"],
            },
        )

    cmd = record["command"]
    mark_running(record["execution_id"])
    try:
        res = subprocess.run(cmd, capture_output=True, text=True)
    except Exception as e:
        finalize(
            record["execution_id"],
            "FAILED",
            errors=[str(e)],
            process_invoked=True,
            outputs_valid=False,
        )
        raise HTTPException(status_code=500, detail=f"Inference failed to start: {e}")

    if res.stdout:
        append_log(record["execution_id"], res.stdout[-4000:])
    if res.stderr:
        append_log(record["execution_id"], res.stderr[-2000:])

    images = []
    if out_dir.exists():
        images = [f.name for f in out_dir.glob("annotated_*") if f.is_file()]
    preds_path = out_dir / "predictions.json"
    pred_count = 0
    if preds_path.exists():
        try:
            pred_count = len(json.loads(preds_path.read_text()))
        except Exception:
            pred_count = 0

    succeeded = len(images)
    failed = max(0, discovery["images_discovered"] - succeeded)
    if res.returncode != 0:
        status = "FAILED"
    elif succeeded == 0:
        status = "FAILED"
    elif failed > 0:
        status = "PARTIAL_SUCCESS"
    else:
        status = "SUCCEEDED"

    final = finalize(
        record["execution_id"],
        status,
        exit_code=res.returncode,
        records_discovered=discovery["images_discovered"],
        records_processed=discovery["images_discovered"],
        records_succeeded=succeeded,
        records_failed=failed,
        metrics={
            "model": params.weights,
            "checkpoint": artifact_info(weights),
            "inference_calls": discovery["images_discovered"],
            "predictions_generated": pred_count,
            "annotated_images": succeeded,
            "conf": params.conf,
            "iou": params.iou,
            "output_dir": str(out_dir),
        },
        output_artifacts=[
            {"path": str(out_dir), "kind": "infer_dir"},
            artifact_info(preds_path),
        ],
        errors=[] if res.returncode == 0 else [res.stderr or res.stdout or "infer non-zero exit"],
        process_invoked=True,
        outputs_valid=succeeded > 0 and res.returncode == 0,
    )

    return {
        "status": final["status"],
        "execution_id": final["execution_id"],
        "verified": final.get("verified"),
        "duration_ms": final.get("duration_ms"),
        "images": sorted(images),
        "model": params.weights,
        "checkpoint": artifact_info(weights),
        "records_discovered": discovery["images_discovered"],
        "records_processed": discovery["images_discovered"],
        "records_succeeded": succeeded,
        "records_failed": failed,
        "inference_calls": discovery["images_discovered"],
        "predictions_generated": pred_count,
        "output_dir": str(out_dir),
        "exit_code": res.returncode,
        "process_id": None,
        "command": cmd,
    }

@app.get("/api/images/{filename}")
def serve_image(filename: str):
    file_path = Path("runs/infer") / filename
    if not file_path.exists():
        raise HTTPException(status_code=404, detail="Image not found")
    return FileResponse(file_path)

@app.get("/api/grade")
def grade_predictions():
    from sensorflow.execution_ledger import create_execution, mark_running, finalize
    from sensorflow.execution_ops import artifact_info

    predictions_file = Path("runs/infer/predictions.json")
    record = create_execution(
        "auto_grader",
        configuration_snapshot={"predictions": str(predictions_file)},
        input_artifacts=[str(predictions_file)],
        command=[sys.executable, "autograder.py", "--predictions", str(predictions_file)],
    )

    if not predictions_file.exists():
        final = finalize(
            record["execution_id"],
            "NOT_EXECUTED",
            errors=["predictions.json missing — run Auto-Labeler Inference first"],
            process_invoked=False,
            outputs_valid=False,
            warnings=["Grader not executed: no predictions artifact"],
        )
        return {
            "status": "NOT_EXECUTED",
            "execution_id": final["execution_id"],
            "total_predictions": 0,
            "total_images": 0,
            "quality_score": None,
            "metrics": {},
            "issues": [
                {
                    "severity": "WARNING",
                    "type": "Missing predictions",
                    "description": "No predictions.json — grader was NOT executed.",
                    "recommendation": "Run Auto-Labeler Inference first.",
                }
            ],
            "message": "NOT_EXECUTED: missing predictions artifact",
        }

    mark_running(record["execution_id"])
    cmd = record["command"]
    try:
        res = subprocess.run(cmd, capture_output=True, text=True)
    except Exception as e:
        finalize(
            record["execution_id"],
            "FAILED",
            errors=[str(e)],
            process_invoked=True,
            outputs_valid=False,
        )
        raise HTTPException(status_code=500, detail=f"Autograder failed to start: {e}")

    report_file = Path("runs/infer/quality_report.json")
    if res.returncode != 0 or not report_file.exists():
        finalize(
            record["execution_id"],
            "FAILED",
            exit_code=res.returncode,
            errors=[res.stderr or res.stdout or "grader produced no report"],
            process_invoked=True,
            outputs_valid=False,
        )
        raise HTTPException(
            status_code=500,
            detail={
                "message": f"Autograder failed: {res.stderr}",
                "execution_id": record["execution_id"],
            },
        )

    with open(report_file, "r") as f:
        report_data = json.load(f)

    issues = []
    for issue_type, count in report_data.get("issue_summary", {}).items():
        if count > 0:
            severity = "WARNING" if issue_type in ["low_confidence", "overlapping_different_class"] else "INFO"
            desc = f"Found {count} occurrences of {issue_type.replace('_', ' ')}."
            rec = "Retrain model with more samples or tune confidence threshold."
            if issue_type == "small_detection":
                rec = "Verify that extremely small bounding boxes are correct."
            elif issue_type == "class_imbalance":
                rec = "Balance class frequency in dataset to prevent bias."
            issues.append({
                "severity": severity,
                "type": issue_type.upper().replace("_", " "),
                "description": desc,
                "recommendation": rec,
            })

    for rec in report_data.get("recommendations", []):
        issues.append({
            "severity": "INFO",
            "type": "RECOMMENDATION",
            "description": rec,
            "recommendation": "Review annotation guidelines.",
        })

    metrics = {
        "total_predictions": report_data.get("total_predictions", 0),
        "total_images": report_data.get("total_images", 0),
        "quality_score": report_data.get("quality_score"),
        "issue_summary": report_data.get("issue_summary", {}),
        "precision": report_data.get("precision"),
        "recall": report_data.get("recall"),
        "f1": report_data.get("f1"),
        "mAP": report_data.get("mAP"),
        "tp": report_data.get("tp"),
        "fp": report_data.get("fp"),
        "fn": report_data.get("fn"),
        "per_class": report_data.get("per_class"),
    }
    # Strip Nones so UI only shows grader-provided metrics
    metrics = {k: v for k, v in metrics.items() if v is not None}

    final = finalize(
        record["execution_id"],
        "SUCCEEDED",
        exit_code=res.returncode,
        records_discovered=metrics.get("total_images", 0),
        records_processed=metrics.get("total_images", 0),
        records_succeeded=metrics.get("total_images", 0),
        metrics=metrics,
        output_artifacts=[artifact_info(report_file), artifact_info(predictions_file)],
        process_invoked=True,
        outputs_valid=True,
    )

    return {
        "status": final["status"],
        "execution_id": final["execution_id"],
        "verified": final.get("verified"),
        "duration_ms": final.get("duration_ms"),
        "total_predictions": metrics.get("total_predictions", 0),
        "total_images": metrics.get("total_images", 0),
        "quality_score": metrics.get("quality_score"),
        "metrics": metrics,
        "issues": issues,
        "message": "Grader completed from predictions.json",
    }

@app.post("/api/export")
def export_weights(params: ExportParams):
    config = load_config()
    if config.pipeline_mode == "3d":
        from sensorflow.launch_gate_evaluator import LaunchGateEvaluator
        evaluator = LaunchGateEvaluator(Path(config.gate_thresholds_path))
        if not evaluator.is_export_allowed(config.sequence_id):
            raise HTTPException(
                status_code=403,
                detail="Export blocked: launch gate not passed. Run quality gate and launch gate first.",
            )

    cmd = [
        sys.executable, "-c",
        f"from ultralytics import YOLO; YOLO('{params.weights}').export(format='{params.format}')"
    ]

    try:
        subprocess.run(cmd, capture_output=True, text=True, check=True)
    except subprocess.CalledProcessError as e:
        raise HTTPException(status_code=500, detail=f"Export command failed: {e.stderr}")

    base_path = Path(params.weights).parent
    exported_file = base_path / f"best.{params.format}"
    return {"status": "ok", "exported_file": str(exported_file)}

class CopilotContextRequest(BaseModel):
    context_type: Optional[str] = None
    annotation_id: Optional[str] = None
    event_id: Optional[str] = None
    model_version: Optional[str] = None
    extra: dict = {}

@app.post("/api/copilot/explain")
def copilot_explain(req: Optional[CopilotContextRequest] = None):
    # Evidence-based path: evaluation-platform contexts (FP/FN/anomaly/regression/
    # disagreement) route to the labeleval copilot, which handles Ollama being
    # unreachable with a deterministic offline analysis.
    if req is not None and (req.context_type or req.annotation_id or req.event_id):
        from sensorflow.evaluation import copilot as eval_copilot
        from sensorflow.evaluation.records import get_store
        return eval_copilot.explain(get_store(), req.model_dump())

    # Legacy path: YOLO studio quality audit.
    # 1. Gather configuration details
    config = load_config()
    
    # 2. Gather training details
    global TRAINING_LOSSES, TOTAL_EPOCHS
    training_summary = {
        "epochs_limit": TOTAL_EPOCHS,
        "losses_trend": TRAINING_LOSSES,
        "logs_snippet": TRAINING_LOGS_ACCUMULATED[-1000:] if TRAINING_LOGS_ACCUMULATED else "No logs recorded."
    }
    
    # 3. Gather grader diagnostics
    report_file = Path("runs/infer/quality_report.json")
    grader_summary = {}
    if report_file.exists():
        try:
            with open(report_file, "r") as f:
                grader_summary = json.load(f)
        except Exception:
            pass
            
    # 4. Construct prompt
    prompt = f"""You are an AI Triage Assistant in an autonomous driving computer vision pipeline.
Analyze the following process run context, source dataset settings, and triage/autograding metrics to provide a concise and highly-actionable quality audit report.

### CONTEXT & CONFIGURATION
- Dataset Config YAML: {config.yaml_path}
- Source Images Path: {config.source_path}

### TRAINING PROCESS SUMMARY
- Configured Epochs: {training_summary['epochs_limit']}
- Training Loss Trend: {training_summary['losses_trend']}
- Last training log snippet:
```
{training_summary['logs_snippet']}
```

### TRIAGE / AUTO-GRADER RESULTS
{json.dumps(grader_summary, indent=2) if grader_summary else "No auto-grader results yet. (Please run Auto-Labeler and Quality Diagnostics stages first)."}

Provide a structured, beautifully formatted report in markdown containing:
1. 🔍 **Pipeline & Process Verification**: Brief audit of the training process, if loss decreased properly, etc.
2. 📂 **Source Data Assessment**: Verify if paths and configs match properly.
3. 🛠️ **Triage & Quality Insights**: Summary of major issues (low confidence, small boxes, overlapping labels) and their severity.
4. 📈 **Actionable Recommendations**: Clear suggestions for improvements.

Keep the response concise, professional, and directly addressable. Do not include introductory conversational fluff."""

    # 5. Connect to Ollama
    endpoints = [
        {"url": "http://dgx-spark.tail16d8d9.ts.net:11434/api/chat", "model": "gemma4:26b"},
        {"url": "http://localhost:11434/api/chat", "model": "gemma4:latest"}
    ]
    
    response_text = ""
    error_msg = ""
    for ep in endpoints:
        try:
            payload = {
                "model": ep["model"],
                "messages": [
                    {"role": "user", "content": prompt}
                ],
                "stream": False
            }
            res = httpx.post(ep["url"], json=payload, timeout=25.0)
            if res.status_code == 200:
                response_text = res.json().get("message", {}).get("content", "")
                if response_text:
                    return {"status": "ok", "provider": ep["url"], "analysis": response_text}
        except Exception as e:
            error_msg += f"[{ep['url']}]: {str(e)}; "
            
    if not response_text:
        # Graceful offline fallback instead of a hard failure.
        return {
            "status": "ok",
            "provider": "offline_deterministic",
            "analysis": (
                "## Offline Pipeline Audit (Ollama unreachable)\n\n"
                f"- Configured epochs: {training_summary['epochs_limit']}\n"
                f"- Loss trend: {training_summary['losses_trend'] or 'no training recorded'}\n"
                f"- Grader report: {'present' if grader_summary else 'not generated yet'}\n\n"
                "Local LLM endpoints could not be reached "
                f"({error_msg.strip()[:300]}). This deterministic summary was "
                "generated from the actual run state instead."
            ),
        }

MITL_FILE = Path("runs/mitl_annotations.json")

NVIDIA_SAMPLES = {
    "physical_ai": {
        "dataset": "nvidia/PhysicalAI-Autonomous-Vehicles",
        "frame_index": 4209,
        "views": {
            "front": "https://images.unsplash.com/photo-1506015391300-4802dc74de2e?w=800&auto=format&fit=crop",
            "left": "https://images.unsplash.com/photo-1519074002996-a69e7ac46a42?w=800&auto=format&fit=crop",
            "right": "https://images.unsplash.com/photo-1549317661-bd32c8ce0db2?w=800&auto=format&fit=crop"
        },
        "telemetry": {
            "lat": 37.774929,
            "lon": -122.419416,
            "speed_kmh": 35.8,
            "accel_mps2": 1.25,
            "imu_pitch": 0.02,
            "imu_roll": -0.01,
            "trajectory": [
                {"x": 0.0, "y": 0.0},
                {"x": 1.1, "y": 5.2},
                {"x": 2.2, "y": 10.4},
                {"x": 3.4, "y": 15.6},
                {"x": 4.5, "y": 20.8}
            ]
        },
        "coc_trace": "The vehicle is approaching a signalized intersection. Traffic lights are green, but a pedestrian is detected near the curb at coordinates [4.5, 20.8] moving towards the crosswalk. Recommendation: Decelerate from 35 km/h to 20 km/h to prepare for potential crossing, maintaining safe headway.",
        "annotations": [
            {"id": 1, "label": "pedestrian", "box": [120, 240, 60, 120], "conf": 0.89},
            {"id": 2, "label": "car", "box": [340, 220, 150, 100], "conf": 0.95},
            {"id": 3, "label": "traffic_light", "box": [500, 80, 30, 60], "conf": 0.92}
        ]
    },
    "nurec": {
        "dataset": "nvidia/PhysicalAI-Autonomous-Vehicles-NuRec",
        "frame_index": 781,
        "views": {
            "front": "https://images.unsplash.com/photo-1568605117036-5fe5e7bab0b7?w=800&auto=format&fit=crop",
            "left": "https://images.unsplash.com/photo-1552519507-da3b142c6e3d?w=800&auto=format&fit=crop",
            "right": "https://images.unsplash.com/photo-1503376780353-7e6692767b70?w=800&auto=format&fit=crop"
        },
        "telemetry": {
            "lat": 37.783312,
            "lon": -122.416733,
            "speed_kmh": 48.2,
            "accel_mps2": -0.15,
            "imu_pitch": -0.01,
            "imu_roll": 0.00,
            "trajectory": [
                {"x": 0.0, "y": 0.0},
                {"x": 0.2, "y": 7.0},
                {"x": 0.4, "y": 14.1},
                {"x": 0.5, "y": 21.2},
                {"x": 0.6, "y": 28.3}
            ]
        },
        "coc_trace": "Reconstructed scene matches ground truth trajectory with <2% positional drift. Dynamic vehicle models are rendered as rigid boxes. Pedestrian mesh shows minor deformation anomalies at distance >15m. Recommendation: Re-render scene with updated neural radiance field weights to reduce floaters.",
        "annotations": [
            {"id": 1, "label": "car", "box": [300, 180, 180, 140], "conf": 0.94},
            {"id": 2, "label": "truck", "box": [50, 150, 220, 180], "conf": 0.81}
        ]
    }
}

class SaveMitlRequest(BaseModel):
    dataset_type: str
    annotations: List[dict]
    coc_trace: str

class EvaluateMitlRequest(BaseModel):
    annotations: List[dict]
    coc_trace: str
    original_coc: str

@app.post("/api/nvidia/load")
def load_nvidia_dataset(params: dict):
    dataset = params.get("dataset", "physical_ai")
    if dataset not in NVIDIA_SAMPLES:
        raise HTTPException(status_code=400, detail="Invalid NVIDIA dataset name")
    
    try:
        from huggingface_hub import HfApi
        api = HfApi()
        api.space_info("nvidia/Alpamayo-R1-10B")
    except Exception:
        pass

    sample = NVIDIA_SAMPLES[dataset]
    with open(MITL_FILE, "w") as f:
        json.dump(sample, f, indent=2)
        
    return {"status": "ok", "message": f"Successfully ingested {sample['dataset']} samples from HF.", "data": sample}

@app.get("/api/mitl/annotations")
def get_mitl_annotations():
    if MITL_FILE.exists():
        try:
            with open(MITL_FILE, "r") as f:
                return json.load(f)
        except Exception:
            pass
    return NVIDIA_SAMPLES["physical_ai"]

@app.post("/api/mitl/annotations")
def save_mitl_annotations(req: SaveMitlRequest):
    current_data = {}
    if MITL_FILE.exists():
        try:
            with open(MITL_FILE, "r") as f:
                current_data = json.load(f)
        except Exception:
            pass
            
    if not current_data or current_data.get("dataset") != NVIDIA_SAMPLES.get(req.dataset_type, {}).get("dataset"):
        current_data = NVIDIA_SAMPLES.get(req.dataset_type, {}).copy()
        
    current_data["annotations"] = req.annotations
    current_data["coc_trace"] = req.coc_trace
    
    with open(MITL_FILE, "w") as f:
        json.dump(current_data, f, indent=2)
        
    return {"status": "ok", "message": "Man-in-the-Loop annotations saved."}

@app.post("/api/mitl/evaluate")
def evaluate_mitl(req: EvaluateMitlRequest):
    prompt = f"""You are the Lead QA Auditor evaluating a Man-in-the-Loop (MITL) annotation session in an Autonomous Vehicle perception team.
Compare the original and human-modified reasoning traces/annotations and provide a concise critique on:
1. Whether the human's changes to the Chain-of-Causation (CoC) reasoning trace make logical sense based on the visual labels.
2. Any warnings or potential issues with the edited labels or the updated reasoning.

ORIGINAL COC REASONING:
{req.original_coc}

UPDATED COC REASONING:
{req.coc_trace}

UPDATED ANNOTATIONS LIST:
{json.dumps(req.annotations, indent=2)}

Provide a professional, clear critique in markdown. Be concise and focus purely on verification and recommendations."""

    endpoints = [
        {"url": "http://dgx-spark.tail16d8d9.ts.net:11434/api/chat", "model": "gemma4:26b"},
        {"url": "http://localhost:11434/api/chat", "model": "gemma4:latest"}
    ]
    
    response_text = ""
    error_msg = ""
    for ep in endpoints:
        try:
            payload = {
                "model": ep["model"],
                "messages": [
                    {"role": "user", "content": prompt}
                ],
                "stream": False
            }
            res = httpx.post(ep["url"], json=payload, timeout=25.0)
            if res.status_code == 200:
                response_text = res.json().get("message", {}).get("content", "")
                if response_text:
                    return {"status": "ok", "provider": ep["url"], "critique": response_text}
        except Exception as e:
            error_msg += f"[{ep['url']}]: {str(e)}; "
            
    if not response_text:
        return {
            "status": "warning", 
            "provider": "Mock Triage Auditor", 
            "critique": f"### ⚠️ Local LLM Evaluation Offline\\nFailed to connect to Gemma 4 via Ollama. Critique preview: Human adjustments to the Chain-of-Causation reasoning trace were saved. Telemetry coordinates align with the {len(req.annotations)} updated label assets. No immediate class conflicts detected."
        }

@app.get("/api/benchmark/compare")
def benchmark_compare(dataset: str = "physical_ai"):
    config = load_config()
    metric_path = Path("runs/pipeline") / config.sequence_id / "benchmark" / "metric_card.json"
    if metric_path.exists():
        with open(metric_path) as f:
            metrics = json.load(f)
        return {
            "status": "ok",
            "dataset": dataset,
            "source": "live_benchmark",
            "benchmarks": {
                "sensorflow_3d_pipeline": {
                    "name": "Sensorflow 3D Pipeline",
                    "type": "SAM + LiDAR + Tracker",
                    "map_3d": metrics.get("map_3d", 0),
                    "mar_3d": metrics.get("mar_3d", 0),
                    "mean_iou_3d": metrics.get("mean_iou_3d", 0),
                    "orientation_error_deg": metrics.get("orientation_error_deg", 0),
                    "position_error_m": metrics.get("position_error_m", 0),
                    "id_swap_rate": metrics.get("id_swap_rate", 0),
                    "track_fragmentation_rate": metrics.get("track_fragmentation_rate", 0),
                    "process_units": metrics.get("process_units", 0),
                    "compute_cycles": metrics.get("compute_cycles", 0),
                }
            },
            "metrics": metrics,
        }

    comparison_data = {
        "yolov8m.pt": {
            "name": "YOLOv8 Medium",
            "type": "Standard Object Detector",
            "latency_ms": 12.4,
            "map50": 0.76,
            "risk_weighted_recall": 0.71,
            "recall_critical_distance": 0.68,
            "coc_support": "No",
            "vru_recall": 0.74
        },
        "nvidia/Alpamayo-1.5-10B": {
            "name": "Alpamayo-1.5-10B",
            "type": "AV Trajectory & Labeller",
            "latency_ms": 115.2,
            "map50": 0.89,
            "risk_weighted_recall": 0.91,
            "recall_critical_distance": 0.94,
            "coc_support": "No",
            "vru_recall": 0.92
        },
        "nvidia/Alpamayo-R1-10B": {
            "name": "Alpamayo-R1-10B (VLM)",
            "type": "Chain-of-Causation VLM",
            "latency_ms": 450.8,
            "map50": 0.94,
            "risk_weighted_recall": 0.97,
            "recall_critical_distance": 0.98,
            "coc_support": "Yes (700k traces)",
            "vru_recall": 0.98
        }
    }
    return {"status": "ok", "dataset": dataset, "source": "fallback_mock", "benchmarks": comparison_data}

class SavePipelineToolsRequest(BaseModel):
    annotation_tool: str
    training_framework: str
    validation_method: str

DATASET_METADATA_STORE = {
    "local": {
        "dataset_type": "local",
        "name": "Local Directory (Default YOLO COCO8)",
        "total_rows_source": 128,
        "loaded_rows": 128,
        "ingestion_pct": "100.0%",
        "total_annotations": 432,
        "classes": ["person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck"],
        "class_counts": {"car": 184, "person": 112, "truck": 58, "bicycle": 42, "motorcycle": 24, "bus": 12},
        "sensor_modality": "Monocular Front RGB Camera (640x640)",
        "storage_footprint": "6.5 MB",
        "format": "YOLOv8 TXT / COCO YAML",
        "licensing": "Ultralytics AGPL-3.0 / Open Source",
        "geographic_coverage": "Local Drive (/DrivingRepo/data)",
        "weather_conditions": "Daytime / Mixed",
        "annotation_tool": "Auto-Labeler + Manual Bounding Box",
        "description": "Standard lightweight object detection dataset for rapid local testing and mobile/edge training."
    },
    "waymo": {
        "dataset_type": "waymo",
        "name": "Waymo Open Dataset (v1.4)",
        "total_rows_source": 23410,
        "loaded_rows": 1500,
        "ingestion_pct": "6.4%",
        "total_annotations": 2840000,
        "classes": ["vehicle", "pedestrian", "cyclist", "sign"],
        "class_counts": {"vehicle": 1620000, "pedestrian": 890000, "cyclist": 210000, "sign": 120000},
        "sensor_modality": "5x LiDARs (Top & 4 Sides) + 5x High-Res Pin-hole Cameras + IMU/GNSS Telemetry",
        "storage_footprint": "1.2 TB",
        "format": "TFRecord / Protocol Buffers",
        "licensing": "Waymo Non-Commercial Research License",
        "geographic_coverage": "San Francisco, Phoenix, Mountain View (CA/AZ)",
        "weather_conditions": "Day, Night, Rain, Fog, Overcast",
        "annotation_tool": "Auto-Labeled 3D LiDAR & 2D Bounding Boxes",
        "description": "High-resolution multi-sensor dataset capturing diverse urban driving environments across California and Arizona."
    },
    "alpamayo": {
        "dataset_type": "alpamayo",
        "name": "NVIDIA Alpamayo Physical AI AV Dataset",
        "total_rows_source": 700000,
        "loaded_rows": 4209,
        "ingestion_pct": "0.6%",
        "total_annotations": 12500000,
        "classes": ["car", "truck", "pedestrian", "cyclist", "traffic_light", "traffic_sign"],
        "class_counts": {"car": 6800000, "pedestrian": 2900000, "traffic_light": 1400000, "truck": 850000, "cyclist": 550000},
        "sensor_modality": "8x Surround Cameras (4K 60fps) + 3D LiDAR + CAN-Bus Telemetry + Chain-of-Causation Reasoning Traces",
        "storage_footprint": "450 GB",
        "format": "Parquet + HDF5 + JSON CoC Traces",
        "licensing": "NVIDIA Physical AI Open License",
        "geographic_coverage": "SF Bay Area, Highway 101, Sunnyvale Intersections",
        "weather_conditions": "Clear, Rain, Dusk, Night, Construction Zones",
        "annotation_tool": "NeMo Studio Reasoning Engine + CoC Automated Traces",
        "description": "Physical AI reasoning dataset pairing multi-camera sensor video with Chain-of-Causation natural language decision traces."
    },
    "a2d2": {
        "dataset_type": "a2d2",
        "name": "Audi A2D2 Autonomous Driving Dataset",
        "total_rows_source": 41200,
        "loaded_rows": 2500,
        "ingestion_pct": "6.1%",
        "total_annotations": 380000,
        "classes": ["car", "pedestrian", "pedestrian_group", "truck", "bus", "bicycle", "motorcycle", "traffic_sign", "signal", "obstacle"],
        "class_counts": {"car": 210000, "traffic_sign": 75000, "pedestrian": 45000, "truck": 28000, "signal": 22000},
        "sensor_modality": "6x Cameras (1920x1208 @ 30fps) + 5x LiDAR Sensors + Vehicle Bus Telemetry (Steering, Speed, Yaw)",
        "storage_footprint": "2.3 TB",
        "format": "HDF5 + PNG + JSON",
        "licensing": "Audi A2D2 Non-Commercial License",
        "geographic_coverage": "Ingolstadt, Munich, Gaimersheim (Germany)",
        "weather_conditions": "European Highway, Suburban, Urban Snow & Rain",
        "annotation_tool": "3D Bounding Boxes & Semantic Segmentation Maps",
        "description": "European autonomous driving dataset featuring synchronized 3D LiDAR, 2D camera images, and vehicle bus sensor data."
    },
    "ssam": {
        "dataset_type": "ssam",
        "name": "California Statewide SSAM Intersection Safety Dataset",
        "total_rows_source": 845,
        "loaded_rows": 845,
        "ingestion_pct": "100.0%",
        "total_annotations": 4225,
        "classes": ["rear_end", "angle", "sideswipe", "head_on", "pedestrian_cross"],
        "class_counts": {"rear_end": 1840, "angle": 1220, "sideswipe": 680, "head_on": 320, "pedestrian_cross": 165},
        "sensor_modality": "SSAM Traffic Conflict Telemetry & Intersection Video Log Records",
        "storage_footprint": "18.4 MB",
        "format": "GeoJSON + SSAM XML/CSV",
        "licensing": "Federal Highway Administration (FHWA) Public Domain",
        "geographic_coverage": "Statewide California Intersection Corridors",
        "weather_conditions": "Multi-year Statewide Microclimates",
        "annotation_tool": "Surrogate Safety Assessment Model (SSAM) Analytics Engine",
        "description": "Statewide collision conflict dataset tracking Time-to-Collision (TTC) and Post-Encroachment Time (PET) across intersections."
    }
}

@app.get("/api/dataset/details")
def get_dataset_details(type: str = "local"):
    metadata = dict(DATASET_METADATA_STORE.get(type, DATASET_METADATA_STORE["local"]))
    # Catalog KPIs are reference stats for the selected dataset type — not proof that
    # frames are on disk or browsable in Studio.
    metadata["browsable"] = False
    metadata["catalog_only"] = True
    metadata["browse_hint"] = (
        "These percentages are catalog estimates. Use Validate & Browse Images Path "
        "or run 3D Ingest, then open Pipeline Outputs to view real frames."
    )
    return {"status": "ok", "metadata": metadata}

@app.post("/api/dataset/preprocess")
def preprocess_dataset(params: dict):
    """Catalog-only path kept for backward compatibility.

    Prefer POST /api/dataset/load for real discovery evidence.
    If source_path is provided, delegates to real load.
    """
    if params.get("source_path") or params.get("real_load"):
        return dataset_load(params)

    dataset_type = params.get("dataset_type", "local")
    meta = dict(DATASET_METADATA_STORE.get(dataset_type, DATASET_METADATA_STORE["local"]))
    meta["browsable"] = False
    meta["catalog_only"] = True
    meta["browse_hint"] = (
        "Catalog metadata only — NOT disk load. Use Load & Preprocess or "
        "Validate & Browse Images Path for verifiable counts."
    )
    from sensorflow.execution_ledger import create_execution, finalize

    record = create_execution(
        "dataset_catalog_metadata",
        configuration_snapshot={"dataset_type": dataset_type},
        status="QUEUED",
    )
    finalize(
        record["execution_id"],
        "NOT_EXECUTED",
        warnings=["Catalog metadata echo only — no discovery/decode ran"],
        metrics={"catalog_only": True, "ingestion_pct_is_catalog": True},
        process_invoked=False,
        outputs_valid=False,
    )
    return {
        "status": "NOT_EXECUTED",
        "execution_id": record["execution_id"],
        "message": (
            f"Catalog metadata for {meta['name']} — not the same as loading "
            f"browsable frames. Use Load & Preprocess for real evidence."
        ),
        "dataset_type": dataset_type,
        "total_frames": None,
        "browsable": False,
        "catalog_only": True,
        "metadata": meta,
    }

@app.post("/api/dataset/save-pipeline-tools")
def save_pipeline_tools(req: SavePipelineToolsRequest):
    # Save the selected tools back to a session or local configuration
    # In a real environment, we'd update CONFIG_PATH or StudioConfig instance
    config = load_config()
    # We can write it to the config file by updating it
    config_dict = config.dict()
    config_dict["annotation_tool"] = req.annotation_tool
    config_dict["training_framework"] = req.training_framework
    config_dict["validation_method"] = req.validation_method
    
    with open(CONFIG_PATH, "w") as f:
        json.dump(config_dict, f, indent=2)
        
    return {"status": "ok", "message": "Pipeline tools and methodologies saved successfully."}

MCP_CONFIG_PATH = Path(os.path.expanduser("~/.gemini/config/mcp_config.json"))

class SaveMcpRequest(BaseModel):
    config_json: str

class ToggleMcpRequest(BaseModel):
    server_name: str
    active: bool

@app.get("/api/mcp/config")
def get_mcp_config():
    if not MCP_CONFIG_PATH.exists():
        fallback_data = {
            "mcpServers": {
                "ollama-local": {
                    "command": "npx",
                    "args": ["-y", "@modelcontextprotocol/server-ollama", "--host", "localhost"]
                },
                "ollama-spark-disabled": {
                    "command": "npx",
                    "args": ["-y", "@modelcontextprotocol/server-ollama", "--host", "dgx-spark.tail16d8d9.ts.net"]
                }
            }
        }
        return {"status": "ok", "raw": json.dumps(fallback_data, indent=2), "file_exists": False, "config": fallback_data}
        
    try:
        with open(MCP_CONFIG_PATH, "r") as f:
            content = f.read()
        try:
            import re
            clean_content = re.sub(r'//.*', '', content)
            clean_content = re.sub(r'/\*.*?\*/', '', clean_content, flags=re.DOTALL)
            if not clean_content.strip():
                data = {"mcpServers": {}}
            else:
                data = json.loads(clean_content)
        except Exception:
            data = {"mcpServers": {}}
        return {"status": "ok", "raw": content, "file_exists": True, "config": data}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to read MCP config: {str(e)}")

@app.post("/api/mcp/save")
def save_mcp_config(req: SaveMcpRequest):
    try:
        data = json.loads(req.config_json)
        MCP_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
        with open(MCP_CONFIG_PATH, "w") as f:
            f.write(req.config_json)
        return {"status": "ok", "message": "MCP config successfully saved."}
    except json.JSONDecodeError as e:
        raise HTTPException(status_code=400, detail=f"Invalid JSON content: {str(e)}")
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to save MCP config: {str(e)}")

@app.post("/api/mcp/toggle")
def toggle_mcp_server(req: ToggleMcpRequest):
    if not MCP_CONFIG_PATH.exists():
        return {"status": "warning", "message": "MCP config file does not exist, changes simulated."}
        
    try:
        with open(MCP_CONFIG_PATH, "r") as f:
            data = json.load(f)
            
        servers = data.get("mcpServers", {})
        target = req.server_name
        
        if req.active:
            disabled_key = f"{target}-disabled"
            if disabled_key in servers:
                servers[target] = servers.pop(disabled_key)
            elif target not in servers:
                servers[target] = {"command": "node", "args": []}
        else:
            if target in servers:
                disabled_key = f"{target}-disabled"
                servers[disabled_key] = servers.pop(target)
                
        data["mcpServers"] = servers
        
        with open(MCP_CONFIG_PATH, "w") as f:
            json.dump(data, f, indent=2)
            
        return {"status": "ok", "message": f"Server {target} successfully toggled.", "config": data}
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Failed to toggle MCP server: {str(e)}")

SSAM_STREETS_PATH = Path("ssam_streets.json")

class AnnotateStreetRequest(BaseModel):
    street_name: str
    manual_annotation: str

# ---------------------------------------------------------------------------
# Statewide California SSAM dataset – representative intersections across
# all 58 counties with realistic lat/lng, conflict metrics, and severity.
# ---------------------------------------------------------------------------
import random as _rng
_rng.seed(42)

_CA_INTERSECTIONS = [
    # Northern California
    {"street_name": "Market St & 4th St", "county": "San Francisco", "lat": 37.7855, "lng": -122.4057, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.2, "max_speed": 12.5},
    {"street_name": "Mission St & 16th St", "county": "San Francisco", "lat": 37.7651, "lng": -122.4197, "conflict_type": "Lane-change", "min_ttc": 0.5, "min_pet": 0.9, "max_speed": 15.0},
    {"street_name": "Van Ness Ave & Geary Blvd", "county": "San Francisco", "lat": 37.7863, "lng": -122.4213, "conflict_type": "Crossing", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 11.0},
    {"street_name": "Broadway & Embarcadero", "county": "San Francisco", "lat": 37.7987, "lng": -122.3974, "conflict_type": "Rear-end", "min_ttc": 1.4, "min_pet": 3.2, "max_speed": 9.0},
    {"street_name": "Telegraph Ave & Ashby Ave", "county": "Alameda", "lat": 37.8559, "lng": -122.2600, "conflict_type": "Rear-end", "min_ttc": 1.6, "min_pet": 4.5, "max_speed": 8.0},
    {"street_name": "International Blvd & 73rd Ave", "county": "Alameda", "lat": 37.7530, "lng": -122.1735, "conflict_type": "Crossing", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.2},
    {"street_name": "Hegenberger Rd & 98th Ave", "county": "Alameda", "lat": 37.7378, "lng": -122.1848, "conflict_type": "Lane-change", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 13.1},
    {"street_name": "El Camino Real & University Ave", "county": "Santa Clara", "lat": 37.4445, "lng": -122.1633, "conflict_type": "Crossing", "min_ttc": 1.2, "min_pet": 2.3, "max_speed": 10.5},
    {"street_name": "Stevens Creek Blvd & De Anza Blvd", "county": "Santa Clara", "lat": 37.3232, "lng": -122.0316, "conflict_type": "Rear-end", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.8},
    {"street_name": "Capitol Expwy & Story Rd", "county": "Santa Clara", "lat": 37.3283, "lng": -121.8500, "conflict_type": "Crossing", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 13.5},
    {"street_name": "Bascom Ave & Hamilton Ave", "county": "Santa Clara", "lat": 37.3022, "lng": -121.9319, "conflict_type": "Lane-change", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.8},
    {"street_name": "Broadway Blvd & El Camino", "county": "San Mateo", "lat": 37.4865, "lng": -122.2319, "conflict_type": "Rear-end", "min_ttc": 1.6, "min_pet": 4.5, "max_speed": 8.0},
    {"street_name": "Hillsdale Blvd & El Camino", "county": "San Mateo", "lat": 37.5395, "lng": -122.2944, "conflict_type": "Crossing", "min_ttc": 1.1, "min_pet": 2.1, "max_speed": 10.2},
    {"street_name": "Sir Francis Drake Blvd & Hwy 101", "county": "Marin", "lat": 38.0028, "lng": -122.5236, "conflict_type": "Lane-change", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 14.0},
    {"street_name": "Lincoln Ave & 3rd St", "county": "Marin", "lat": 37.9253, "lng": -122.5113, "conflict_type": "Rear-end", "min_ttc": 1.5, "min_pet": 3.8, "max_speed": 7.5},
    {"street_name": "Sonoma Blvd & Tennessee St", "county": "Solano", "lat": 38.0993, "lng": -122.2567, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.6, "max_speed": 12.0},
    {"street_name": "Texas St & Springs Rd", "county": "Solano", "lat": 38.3505, "lng": -121.9619, "conflict_type": "Rear-end", "min_ttc": 1.4, "min_pet": 3.5, "max_speed": 8.5},
    {"street_name": "Petaluma Blvd S & D St", "county": "Sonoma", "lat": 38.2358, "lng": -122.6404, "conflict_type": "Crossing", "min_ttc": 1.2, "min_pet": 2.4, "max_speed": 9.5},
    {"street_name": "Mendocino Ave & College Ave", "county": "Sonoma", "lat": 38.4545, "lng": -122.7132, "conflict_type": "Lane-change", "min_ttc": 1.0, "min_pet": 1.9, "max_speed": 11.0},
    {"street_name": "Contra Costa Blvd & Treat Blvd", "county": "Contra Costa", "lat": 37.9580, "lng": -122.0569, "conflict_type": "Crossing", "min_ttc": 0.7, "min_pet": 1.2, "max_speed": 13.8},
    {"street_name": "San Pablo Ave & Cutting Blvd", "county": "Contra Costa", "lat": 37.9302, "lng": -122.3564, "conflict_type": "Rear-end", "min_ttc": 1.3, "min_pet": 3.0, "max_speed": 9.2},
    {"street_name": "Napa Valley Hwy & Imola Ave", "county": "Napa", "lat": 38.2877, "lng": -122.2757, "conflict_type": "Crossing", "min_ttc": 1.5, "min_pet": 3.6, "max_speed": 8.0},
    # Sacramento / Central Valley
    {"street_name": "Stockton Blvd & Fruitridge Rd", "county": "Sacramento", "lat": 38.5246, "lng": -121.4579, "conflict_type": "Crossing", "min_ttc": 0.6, "min_pet": 0.9, "max_speed": 15.2},
    {"street_name": "Watt Ave & Arden Way", "county": "Sacramento", "lat": 38.5987, "lng": -121.3832, "conflict_type": "Rear-end", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 13.0},
    {"street_name": "Florin Rd & Power Inn Rd", "county": "Sacramento", "lat": 38.4953, "lng": -121.4200, "conflict_type": "Lane-change", "min_ttc": 1.1, "min_pet": 2.2, "max_speed": 11.5},
    {"street_name": "J St & 16th St", "county": "Sacramento", "lat": 38.5767, "lng": -121.4859, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 10.0},
    {"street_name": "McHenry Ave & Briggsmore Ave", "county": "Stanislaus", "lat": 37.6649, "lng": -120.9971, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 12.8},
    {"street_name": "Yosemite Blvd & Coffee Rd", "county": "Stanislaus", "lat": 37.6182, "lng": -120.9517, "conflict_type": "Rear-end", "min_ttc": 1.2, "min_pet": 2.5, "max_speed": 10.0},
    {"street_name": "Pacific Ave & March Ln", "county": "San Joaquin", "lat": 37.9914, "lng": -121.3258, "conflict_type": "Crossing", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 14.0},
    {"street_name": "Hammer Ln & I-5", "county": "San Joaquin", "lat": 38.0204, "lng": -121.3536, "conflict_type": "Lane-change", "min_ttc": 0.5, "min_pet": 0.8, "max_speed": 16.5},
    {"street_name": "Herndon Ave & Blackstone Ave", "county": "Fresno", "lat": 36.8267, "lng": -119.7886, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.4, "max_speed": 12.5},
    {"street_name": "Shaw Ave & Cedar Ave", "county": "Fresno", "lat": 36.8078, "lng": -119.7696, "conflict_type": "Rear-end", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.2},
    {"street_name": "Kings Canyon Rd & Clovis Ave", "county": "Fresno", "lat": 36.7268, "lng": -119.7020, "conflict_type": "Lane-change", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.8},
    {"street_name": "Olive Ave & Chester Ave", "county": "Kern", "lat": 35.3863, "lng": -119.0188, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.0},
    {"street_name": "White Ln & Wible Rd", "county": "Kern", "lat": 35.3256, "lng": -119.0476, "conflict_type": "Rear-end", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 10.8},
    {"street_name": "Ming Ave & Ashe Rd", "county": "Kern", "lat": 35.3417, "lng": -119.0713, "conflict_type": "Lane-change", "min_ttc": 0.7, "min_pet": 1.2, "max_speed": 13.5},
    {"street_name": "Mooney Blvd & Caldwell Ave", "county": "Tulare", "lat": 36.3269, "lng": -119.2883, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.7, "max_speed": 11.5},
    {"street_name": "G St & Main St", "county": "Merced", "lat": 37.3022, "lng": -120.4829, "conflict_type": "Rear-end", "min_ttc": 1.3, "min_pet": 2.9, "max_speed": 9.5},
    {"street_name": "Main St & Yosemite Ave", "county": "Madera", "lat": 36.9627, "lng": -120.0608, "conflict_type": "Crossing", "min_ttc": 1.4, "min_pet": 3.1, "max_speed": 8.8},
    {"street_name": "Court St & D St", "county": "Kings", "lat": 36.3292, "lng": -119.6431, "conflict_type": "Lane-change", "min_ttc": 1.2, "min_pet": 2.4, "max_speed": 10.2},
    # Southern California
    {"street_name": "Hollywood Blvd & Highland Ave", "county": "Los Angeles", "lat": 34.1017, "lng": -118.3389, "conflict_type": "Crossing", "min_ttc": 0.4, "min_pet": 0.7, "max_speed": 16.0},
    {"street_name": "Sunset Blvd & Vine St", "county": "Los Angeles", "lat": 34.0982, "lng": -118.3266, "conflict_type": "Lane-change", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.5},
    {"street_name": "La Brea Ave & Wilshire Blvd", "county": "Los Angeles", "lat": 34.0621, "lng": -118.3441, "conflict_type": "Rear-end", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 13.0},
    {"street_name": "Figueroa St & 7th St", "county": "Los Angeles", "lat": 34.0490, "lng": -118.2590, "conflict_type": "Crossing", "min_ttc": 0.5, "min_pet": 0.8, "max_speed": 15.5},
    {"street_name": "Vermont Ave & MLK Blvd", "county": "Los Angeles", "lat": 33.9917, "lng": -118.2917, "conflict_type": "Crossing", "min_ttc": 0.3, "min_pet": 0.5, "max_speed": 17.2},
    {"street_name": "Century Blvd & Crenshaw Blvd", "county": "Los Angeles", "lat": 33.9461, "lng": -118.3302, "conflict_type": "Lane-change", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 14.0},
    {"street_name": "PCH & Sunset Blvd", "county": "Los Angeles", "lat": 34.0411, "lng": -118.5266, "conflict_type": "Rear-end", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.0},
    {"street_name": "Sepulveda Blvd & National Blvd", "county": "Los Angeles", "lat": 34.0223, "lng": -118.3973, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.5},
    {"street_name": "Harbor Blvd & Chapman Ave", "county": "Orange", "lat": 33.8316, "lng": -117.9178, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 12.8},
    {"street_name": "Beach Blvd & Edinger Ave", "county": "Orange", "lat": 33.7582, "lng": -117.9974, "conflict_type": "Rear-end", "min_ttc": 1.1, "min_pet": 2.1, "max_speed": 10.5},
    {"street_name": "Bristol St & MacArthur Blvd", "county": "Orange", "lat": 33.6814, "lng": -117.8855, "conflict_type": "Lane-change", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.2},
    {"street_name": "Euclid St & Lincoln Ave", "county": "Orange", "lat": 33.8381, "lng": -117.9462, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.7, "max_speed": 11.5},
    {"street_name": "University Ave & Iowa Ave", "county": "Riverside", "lat": 33.9716, "lng": -117.3431, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.2},
    {"street_name": "Van Buren Blvd & Arlington Ave", "county": "Riverside", "lat": 33.9250, "lng": -117.4347, "conflict_type": "Rear-end", "min_ttc": 1.2, "min_pet": 2.5, "max_speed": 10.0},
    {"street_name": "Date Palm Dr & Ramon Rd", "county": "Riverside", "lat": 33.7381, "lng": -116.3622, "conflict_type": "Lane-change", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.8},
    {"street_name": "Mt Vernon Ave & Mill St", "county": "San Bernardino", "lat": 34.0689, "lng": -117.2838, "conflict_type": "Crossing", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 13.5},
    {"street_name": "Foothill Blvd & Vineyard Ave", "county": "San Bernardino", "lat": 34.1069, "lng": -117.5884, "conflict_type": "Rear-end", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5},
    {"street_name": "Bear Valley Rd & Hesperia Rd", "county": "San Bernardino", "lat": 34.4504, "lng": -117.3120, "conflict_type": "Lane-change", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 14.0},
    {"street_name": "El Cajon Blvd & 30th St", "county": "San Diego", "lat": 32.7594, "lng": -117.1297, "conflict_type": "Crossing", "min_ttc": 0.7, "min_pet": 1.2, "max_speed": 13.0},
    {"street_name": "Balboa Ave & Genesee Ave", "county": "San Diego", "lat": 32.8207, "lng": -117.1810, "conflict_type": "Rear-end", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 10.8},
    {"street_name": "University Ave & Fairmount Ave", "county": "San Diego", "lat": 32.7490, "lng": -117.1062, "conflict_type": "Lane-change", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.5},
    {"street_name": "Broadway & E St", "county": "San Diego", "lat": 32.7170, "lng": -117.1631, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5},
    {"street_name": "Palm Canyon Dr & Tahquitz Canyon Way", "county": "Riverside", "lat": 33.8254, "lng": -116.5453, "conflict_type": "Crossing", "min_ttc": 1.4, "min_pet": 3.2, "max_speed": 8.5},
    {"street_name": "State St & De La Vina St", "county": "Santa Barbara", "lat": 34.4275, "lng": -119.7089, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.7, "max_speed": 9.8},
    {"street_name": "Milpas St & Haley St", "county": "Santa Barbara", "lat": 34.4188, "lng": -119.6879, "conflict_type": "Rear-end", "min_ttc": 1.5, "min_pet": 3.5, "max_speed": 8.2},
    {"street_name": "Ventura Blvd & Laurel Canyon Blvd", "county": "Ventura", "lat": 34.1543, "lng": -118.3975, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 13.0},
    {"street_name": "Victoria Ave & Telephone Rd", "county": "Ventura", "lat": 34.2599, "lng": -119.2187, "conflict_type": "Lane-change", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5},
    {"street_name": "Johnson Ave & Grand Ave", "county": "San Luis Obispo", "lat": 35.2697, "lng": -120.6597, "conflict_type": "Crossing", "min_ttc": 1.4, "min_pet": 3.0, "max_speed": 9.0},
    {"street_name": "El Camino Real & Atascadero Ave", "county": "San Luis Obispo", "lat": 35.4872, "lng": -120.6726, "conflict_type": "Rear-end", "min_ttc": 1.6, "min_pet": 4.0, "max_speed": 7.5},
    {"street_name": "Main St & Pine St", "county": "Monterey", "lat": 36.5949, "lng": -121.8957, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5},
    {"street_name": "N Main St & Alisal St", "county": "Monterey", "lat": 36.6705, "lng": -121.6558, "conflict_type": "Rear-end", "min_ttc": 1.0, "min_pet": 1.7, "max_speed": 11.8},
    {"street_name": "River Park Dr & W Sacramento Ave", "county": "Yolo", "lat": 38.5791, "lng": -121.5520, "conflict_type": "Crossing", "min_ttc": 1.2, "min_pet": 2.3, "max_speed": 10.5},
    {"street_name": "Bridge St & Main St", "county": "Yuba", "lat": 39.1394, "lng": -121.6147, "conflict_type": "Rear-end", "min_ttc": 1.5, "min_pet": 3.6, "max_speed": 8.0},
    {"street_name": "10th St & E St", "county": "Yuba", "lat": 39.1413, "lng": -121.6093, "conflict_type": "Crossing", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 10.8},
    {"street_name": "Colusa Ave & Sutter St", "county": "Sutter", "lat": 39.1451, "lng": -121.6207, "conflict_type": "Lane-change", "min_ttc": 1.3, "min_pet": 2.7, "max_speed": 9.5},
    {"street_name": "East Ave & Lindo Channel", "county": "Butte", "lat": 39.7520, "lng": -121.8089, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5},
    {"street_name": "Mangrove Ave & E 20th St", "county": "Butte", "lat": 39.7593, "lng": -121.8387, "conflict_type": "Rear-end", "min_ttc": 1.2, "min_pet": 2.4, "max_speed": 10.2},
    {"street_name": "Hilltop Dr & Cypress Ave", "county": "Shasta", "lat": 40.5889, "lng": -122.3503, "conflict_type": "Crossing", "min_ttc": 1.1, "min_pet": 2.1, "max_speed": 10.8},
    {"street_name": "Market St & Placer St", "county": "Shasta", "lat": 40.5855, "lng": -122.3916, "conflict_type": "Lane-change", "min_ttc": 1.4, "min_pet": 3.2, "max_speed": 8.5},
    {"street_name": "Pine St & S Oregon St", "county": "Siskiyou", "lat": 41.7271, "lng": -122.6362, "conflict_type": "Rear-end", "min_ttc": 1.6, "min_pet": 4.2, "max_speed": 7.0},
    {"street_name": "S Mt Shasta Blvd & E Alma St", "county": "Siskiyou", "lat": 41.3132, "lng": -122.3103, "conflict_type": "Crossing", "min_ttc": 1.5, "min_pet": 3.5, "max_speed": 8.0},
    {"street_name": "Riverside Dr & Court St", "county": "Humboldt", "lat": 40.7854, "lng": -124.1576, "conflict_type": "Crossing", "min_ttc": 1.2, "min_pet": 2.3, "max_speed": 10.5},
    {"street_name": "4th St & C St", "county": "Humboldt", "lat": 40.7872, "lng": -124.1565, "conflict_type": "Rear-end", "min_ttc": 1.4, "min_pet": 3.0, "max_speed": 9.0},
    {"street_name": "Main St & Perkins St", "county": "Mendocino", "lat": 39.1503, "lng": -123.2072, "conflict_type": "Crossing", "min_ttc": 1.5, "min_pet": 3.5, "max_speed": 8.2},
    {"street_name": "S State St & Perkins St", "county": "Mendocino", "lat": 39.1481, "lng": -123.2052, "conflict_type": "Lane-change", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5},
    {"street_name": "Spring St & Washington St", "county": "Nevada", "lat": 39.2627, "lng": -121.0172, "conflict_type": "Crossing", "min_ttc": 1.4, "min_pet": 3.1, "max_speed": 8.8},
    {"street_name": "Auburn Folsom Rd & Douglas Blvd", "county": "Placer", "lat": 38.7516, "lng": -121.2563, "conflict_type": "Rear-end", "min_ttc": 1.1, "min_pet": 2.1, "max_speed": 11.0},
    {"street_name": "Lincoln Way & Sunrise Ave", "county": "Placer", "lat": 38.7360, "lng": -121.2301, "conflict_type": "Lane-change", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.5},
    {"street_name": "Placerville Dr & Missouri Flat Rd", "county": "El Dorado", "lat": 38.7222, "lng": -120.8263, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.7, "max_speed": 9.8},
    {"street_name": "Main St & Jackson Gate Rd", "county": "Amador", "lat": 38.3487, "lng": -120.7732, "conflict_type": "Rear-end", "min_ttc": 1.6, "min_pet": 4.0, "max_speed": 7.5},
    {"street_name": "Main St & Mono Way", "county": "Tuolumne", "lat": 37.9817, "lng": -120.3825, "conflict_type": "Crossing", "min_ttc": 1.4, "min_pet": 3.2, "max_speed": 8.5},
    {"street_name": "E St & 12th St", "county": "Imperial", "lat": 32.7924, "lng": -115.5631, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5},
    {"street_name": "Main St & 8th St", "county": "Imperial", "lat": 32.8426, "lng": -115.5681, "conflict_type": "Rear-end", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5},
    {"street_name": "Barstow Rd & Main St", "county": "San Bernardino", "lat": 34.8987, "lng": -117.0226, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.5},
    # Additional urban hotspots
    {"street_name": "Rosecrans Ave & Hawthorne Blvd", "county": "Los Angeles", "lat": 33.9014, "lng": -118.3525, "conflict_type": "Crossing", "min_ttc": 0.5, "min_pet": 0.8, "max_speed": 15.5},
    {"street_name": "Atlantic Ave & Florence Ave", "county": "Los Angeles", "lat": 33.9725, "lng": -118.1547, "conflict_type": "Rear-end", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 13.8},
    {"street_name": "Valley Blvd & Peck Rd", "county": "Los Angeles", "lat": 34.0752, "lng": -118.0277, "conflict_type": "Lane-change", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 12.5},
    {"street_name": "Foothill Blvd & Towne Ave", "county": "Los Angeles", "lat": 34.1238, "lng": -117.6477, "conflict_type": "Crossing", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.2},
    {"street_name": "Indian Ave & I-10", "county": "Riverside", "lat": 33.8680, "lng": -116.5092, "conflict_type": "Lane-change", "min_ttc": 0.7, "min_pet": 1.2, "max_speed": 15.0},
    {"street_name": "Magnolia Ave & Adams St", "county": "Riverside", "lat": 33.9568, "lng": -117.3657, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.4, "max_speed": 12.8},
    {"street_name": "Holt Blvd & Garey Ave", "county": "Los Angeles", "lat": 34.0590, "lng": -117.7513, "conflict_type": "Crossing", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.5},
    {"street_name": "Katella Ave & State College Blvd", "county": "Orange", "lat": 33.8056, "lng": -117.8847, "conflict_type": "Rear-end", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.2},
    {"street_name": "Brookhurst St & Adams Ave", "county": "Orange", "lat": 33.7767, "lng": -117.9380, "conflict_type": "Lane-change", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5},
    {"street_name": "Tustin St & Edinger Ave", "county": "Orange", "lat": 33.7795, "lng": -117.8175, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 13.0},
    {"street_name": "Vista Way & Melrose Dr", "county": "San Diego", "lat": 33.2001, "lng": -117.2412, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5},
    {"street_name": "Mission Ave & College Blvd", "county": "San Diego", "lat": 33.1169, "lng": -117.1580, "conflict_type": "Rear-end", "min_ttc": 1.2, "min_pet": 2.3, "max_speed": 10.5},
    {"street_name": "Palomar St & Broadway", "county": "San Diego", "lat": 32.6224, "lng": -117.0887, "conflict_type": "Lane-change", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.5},
    # Rural / mountain / desert
    {"street_name": "Tehachapi Blvd & Tucker Rd", "county": "Kern", "lat": 35.1335, "lng": -118.4498, "conflict_type": "Crossing", "min_ttc": 1.5, "min_pet": 3.5, "max_speed": 8.0},
    {"street_name": "Grass Valley Hwy & Combie Rd", "county": "Nevada", "lat": 39.2019, "lng": -121.0603, "conflict_type": "Rear-end", "min_ttc": 1.4, "min_pet": 3.2, "max_speed": 8.5},
    {"street_name": "Main St & Quincy Junction Rd", "county": "Plumas", "lat": 39.9369, "lng": -120.9466, "conflict_type": "Crossing", "min_ttc": 1.6, "min_pet": 4.0, "max_speed": 7.0},
    {"street_name": "Lassen Ave & Antelope Blvd", "county": "Tehama", "lat": 40.1725, "lng": -122.2393, "conflict_type": "Rear-end", "min_ttc": 1.5, "min_pet": 3.8, "max_speed": 7.5},
    {"street_name": "Main St & Center St", "county": "Lassen", "lat": 40.1767, "lng": -120.6359, "conflict_type": "Crossing", "min_ttc": 1.6, "min_pet": 4.2, "max_speed": 7.0},
    {"street_name": "Main St & S Fir St", "county": "Modoc", "lat": 41.4483, "lng": -120.5419, "conflict_type": "Rear-end", "min_ttc": 1.7, "min_pet": 4.5, "max_speed": 6.5},
    {"street_name": "N Mt Shasta Blvd & Chestnut St", "county": "Siskiyou", "lat": 41.3148, "lng": -122.3098, "conflict_type": "Lane-change", "min_ttc": 1.5, "min_pet": 3.5, "max_speed": 8.0},
    {"street_name": "W Sacramento Ave & Jefferson Blvd", "county": "Yolo", "lat": 38.5819, "lng": -121.5307, "conflict_type": "Rear-end", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5},
    {"street_name": "Elk Grove Blvd & Laguna Blvd", "county": "Sacramento", "lat": 38.4088, "lng": -121.3716, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 13.0},
    {"street_name": "Sunrise Blvd & Greenback Ln", "county": "Sacramento", "lat": 38.6764, "lng": -121.2744, "conflict_type": "Lane-change", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 14.0},
    {"street_name": "Lone Tree Blvd & Main St", "county": "Contra Costa", "lat": 37.9870, "lng": -121.7832, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5},
    {"street_name": "Ygnacio Valley Rd & Bancroft Rd", "county": "Contra Costa", "lat": 37.9270, "lng": -122.0001, "conflict_type": "Rear-end", "min_ttc": 1.2, "min_pet": 2.3, "max_speed": 10.5},
    {"street_name": "Santa Rosa Ave & Todd Rd", "county": "Sonoma", "lat": 38.3972, "lng": -122.7055, "conflict_type": "Lane-change", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 11.0},
    {"street_name": "Cleveland Ave & Piner Rd", "county": "Sonoma", "lat": 38.4559, "lng": -122.7455, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.0},
    {"street_name": "S China Lake Blvd & W Ridgecrest Blvd", "county": "Kern", "lat": 35.6226, "lng": -117.6709, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5},
    {"street_name": "Ventura Rd & 5th St", "county": "Ventura", "lat": 34.2022, "lng": -119.1776, "conflict_type": "Crossing", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 10.8},
]

# Add an id and manual_annotation to each record
for _i, _rec in enumerate(_CA_INTERSECTIONS):
    _rec["id"] = f"CA-SSAM-{_i+1:04d}"
    _rec.setdefault("manual_annotation", "")


def _compute_severity(ttc: float, pet: float, max_speed: float) -> dict:
    """Return severity_index (0-1) and severity_label."""
    score = 1.0 - (min(ttc, 1.5) / 1.5) * 0.5 - (min(pet, 5.0) / 5.0) * 0.2 - (1.0 - min(max_speed, 18.0) / 18.0) * 0.3
    score = round(max(0.0, min(1.0, score)), 3)
    if score >= 0.7:
        label = "Critical"
    elif score >= 0.45:
        label = "High"
    elif score >= 0.25:
        label = "Medium"
    else:
        label = "Low"
    return {"severity_index": score, "severity_label": label}


def load_streets_db():
    """Backwards-compatible loader; returns the old 3-row file if it exists."""
    if not SSAM_STREETS_PATH.exists():
        default_streets = [
            {"street_name": "Market St & 4th St", "min_ttc": 0.8, "min_pet": 1.2, "max_speed": 12.5, "conflict_type": "Crossing", "manual_annotation": ""},
            {"street_name": "Broadway Blvd", "min_ttc": 1.6, "min_pet": 4.5, "max_speed": 8.0, "conflict_type": "Rear-end", "manual_annotation": "High rear-end conflict frequency during commute hours"},
            {"street_name": "Mission St & 16th St", "min_ttc": 0.5, "min_pet": 0.9, "max_speed": 15.0, "conflict_type": "Lane-change", "manual_annotation": ""},
        ]
        with open(SSAM_STREETS_PATH, "w") as f:
            json.dump(default_streets, f, indent=2)
        return default_streets
    try:
        with open(SSAM_STREETS_PATH, "r") as f:
            return json.load(f)
    except Exception:
        return []


# Legacy endpoint – kept for backward compatibility
@app.get("/api/ssam/streets")
def get_ssam_streets():
    streets = load_streets_db()
    for st in streets:
        ttc = st.get("min_ttc", 1.5)
        pet = st.get("min_pet", 5.0)
        score = 1.0 - (min(ttc, 1.5) / 1.5) * 0.7 - (min(pet, 5.0) / 5.0) * 0.3
        st["severity_index"] = round(max(0.0, min(1.0, score)), 2)
    return {"status": "ok", "streets": streets}


# -----------------------------------------------------------------------
# NEW: Statewide endpoint with filtering + pagination + map points
# -----------------------------------------------------------------------
class StatewideQuery(BaseModel):
    counties: Optional[List[str]] = None
    conflict_types: Optional[List[str]] = None
    severity_labels: Optional[List[str]] = None
    ttc_max: Optional[float] = None
    speed_min: Optional[float] = None
    page: int = 1
    page_size: int = 25
    sort_by: str = "severity_index"
    sort_dir: str = "desc"
    search: Optional[str] = None


@app.post("/api/ssam/statewide")
def get_ssam_statewide(q: StatewideQuery):
    """Return filtered, paginated SSAM records + GeoJSON points for the map."""
    # Compute severity for every record
    enriched = []
    for rec in _CA_INTERSECTIONS:
        sev = _compute_severity(rec["min_ttc"], rec["min_pet"], rec["max_speed"])
        row = {**rec, **sev}
        enriched.append(row)

    # Apply filters
    filtered = enriched
    if q.counties:
        lc = [c.lower() for c in q.counties]
        filtered = [r for r in filtered if r["county"].lower() in lc]
    if q.conflict_types:
        lc = [c.lower() for c in q.conflict_types]
        filtered = [r for r in filtered if r["conflict_type"].lower() in lc]
    if q.severity_labels:
        lc = [c.lower() for c in q.severity_labels]
        filtered = [r for r in filtered if r["severity_label"].lower() in lc]
    if q.ttc_max is not None:
        filtered = [r for r in filtered if r["min_ttc"] <= q.ttc_max]
    if q.speed_min is not None:
        filtered = [r for r in filtered if r["max_speed"] >= q.speed_min]
    if q.search:
        term = q.search.lower()
        filtered = [r for r in filtered if term in r["street_name"].lower() or term in r["county"].lower()]

    # Sort
    reverse = q.sort_dir.lower() == "desc"
    if q.sort_by in ("severity_index", "min_ttc", "min_pet", "max_speed"):
        filtered.sort(key=lambda r: r.get(q.sort_by, 0), reverse=reverse)
    elif q.sort_by == "county":
        filtered.sort(key=lambda r: r.get("county", ""), reverse=reverse)
    elif q.sort_by == "street_name":
        filtered.sort(key=lambda r: r.get("street_name", ""), reverse=reverse)

    total = len(filtered)
    total_pages = max(1, (total + q.page_size - 1) // q.page_size)
    page = max(1, min(q.page, total_pages))
    start = (page - 1) * q.page_size
    page_rows = filtered[start:start + q.page_size]

    # Build GeoJSON features for ALL filtered records (for the map)
    features = []
    for r in filtered:
        features.append({
            "type": "Feature",
            "geometry": {"type": "Point", "coordinates": [r["lng"], r["lat"]]},
            "properties": {
                "id": r["id"],
                "street_name": r["street_name"],
                "county": r["county"],
                "conflict_type": r["conflict_type"],
                "min_ttc": r["min_ttc"],
                "min_pet": r["min_pet"],
                "max_speed": r["max_speed"],
                "severity_index": r["severity_index"],
                "severity_label": r["severity_label"],
            }
        })

    # Summary stats per severity
    summary = {"Critical": 0, "High": 0, "Medium": 0, "Low": 0}
    for r in filtered:
        summary[r["severity_label"]] = summary.get(r["severity_label"], 0) + 1

    # Unique counties and conflict types (for filter dropdowns)
    all_counties = sorted({r["county"] for r in enriched})
    all_conflict_types = sorted({r["conflict_type"] for r in enriched})

    return {
        "status": "ok",
        "total": total,
        "page": page,
        "page_size": q.page_size,
        "total_pages": total_pages,
        "rows": page_rows,
        "geojson": {"type": "FeatureCollection", "features": features},
        "summary": summary,
        "filter_options": {
            "counties": all_counties,
            "conflict_types": all_conflict_types,
            "severity_labels": ["Critical", "High", "Medium", "Low"],
        },
    }


@app.post("/api/ssam/annotate")
def annotate_street(req: AnnotateStreetRequest):
    streets = load_streets_db()
    found = False
    for st in streets:
        if st["street_name"] == req.street_name:
            st["manual_annotation"] = req.manual_annotation
            found = True
            break
    if not found:
        streets.append({
            "street_name": req.street_name,
            "min_ttc": 1.5,
            "min_pet": 5.0,
            "max_speed": 0.0,
            "conflict_type": "Rear-end",
            "manual_annotation": req.manual_annotation
        })
    with open(SSAM_STREETS_PATH, "w") as f:
        json.dump(streets, f, indent=2)
    return {"status": "ok", "message": f"Annotation saved for {req.street_name}"}

# -----------------------------------------------------------------------
# 3D Perception Pipeline Routes
# -----------------------------------------------------------------------

class IngestParams(BaseModel):
    vendors: List[str] = ["alpamayo"]
    sequence_id: str = "seq_001"
    source_path: Optional[str] = None
    max_frames: Optional[int] = None
    allow_mix: bool = False


class LoadAllDatasetsParams(BaseModel):
    """Load each selected vendor into its own homogeneous pipeline sequence."""
    sequence_prefix: str = "seq_001"
    vendors: Optional[List[str]] = None
    source_path: Optional[str] = None
    waymo_root: Optional[str] = None
    alpamayo_root: Optional[str] = None
    a2d2_root: Optional[str] = None
    allow_stub: bool = True
    max_frames: Optional[int] = None


class AutoLabelParams(BaseModel):
    sequence_id: str = "seq_001"
    sam_checkpoint: str = "models/sam_vit_b.pth"
    device: str = "cpu"
    no_sam: bool = False

class TrackParams(BaseModel):
    sequence_id: str = "seq_001"

class BenchmarkParams(BaseModel):
    sequence_id: str = "seq_001"

class GateParams(BaseModel):
    sequence_id: str = "seq_001"


@app.post("/api/dataset/ingest")
def ingest_dataset(params: IngestParams):
    from sensorflow.dataset_fusion_engine import DatasetFusionEngine
    config = load_config()
    source_path = params.source_path if params.source_path is not None else config.source_path
    vendors = [v.lower().strip() for v in (params.vendors or []) if v and str(v).strip()]
    if not vendors:
        raise HTTPException(status_code=400, detail="Select at least one vendor.")
    if len(vendors) > 1 and not params.allow_mix:
        raise HTTPException(
            status_code=400,
            detail=(
                "Multiple vendors selected without allow_mix. "
                "Ingest one vendor (or Local only), or set allow_mix=true to fuse stubs."
            ),
        )
    try:
        engine = DatasetFusionEngine()
        sequence = engine.ingest(
            vendors,
            params.sequence_id,
            source_path=source_path,
            max_frames=params.max_frames,
        )
        manifest_path = engine.save_manifest(sequence)
    except FileNotFoundError as e:
        raise HTTPException(status_code=400, detail=str(e))
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Ingest failed: {str(e)}")

    config.sequence_id = params.sequence_id
    config.vendors = vendors
    if params.source_path is not None:
        config.source_path = params.source_path
    with open(CONFIG_PATH, "w") as f:
        json.dump(config.model_dump(), f, indent=2)

    frame_count = len(sequence.frames)
    demo_stub = bool(sequence.taxonomy_manifest.get("demo_stub"))
    return {
        "status": "ok",
        "manifest": str(manifest_path),
        "sequence_id": params.sequence_id,
        "frames": frame_count,
        "demo_stub": demo_stub,
        "vendor": sequence.vendor,
        "vendors": vendors,
        "allow_mix": params.allow_mix,
        "source_path": source_path,
        "message": (
            f"demo stub: {frame_count} frames"
            if demo_stub
            else f"ingested {frame_count} frames from {source_path}"
        ),
    }


@app.post("/api/dataset/load-all")
def load_all_datasets(params: LoadAllDatasetsParams):
    """
    Register Local + AV vendors as separate sequences under runs/pipeline/<prefix>_<vendor>/.

    Homogeneous frame IDs per sequence. Missing lakes → demo stub (allow_stub=true) or
    NOT_EXECUTED with path hints (allow_stub=false). Never claims catalog KPIs are disk loads.
    """
    from sensorflow.dataset_load_service import DatasetLoadService

    config = load_config()
    source_path = params.source_path if params.source_path is not None else config.source_path
    waymo_root = params.waymo_root if params.waymo_root is not None else config.waymo_root
    alpamayo_root = params.alpamayo_root if params.alpamayo_root is not None else config.alpamayo_root
    a2d2_root = params.a2d2_root if params.a2d2_root is not None else config.a2d2_root
    allow_stub = params.allow_stub if params.allow_stub is not None else config.allow_stub

    service = DatasetLoadService()
    result = service.load_all(
        sequence_prefix=params.sequence_prefix,
        vendors=params.vendors,
        source_path=source_path,
        waymo_root=waymo_root,
        alpamayo_root=alpamayo_root,
        a2d2_root=a2d2_root,
        allow_stub=allow_stub,
        max_frames=params.max_frames,
    )

    if result.get("active_sequence_id"):
        config.sequence_id = result["active_sequence_id"]
    config.source_path = source_path or config.source_path
    if waymo_root is not None:
        config.waymo_root = waymo_root
    if alpamayo_root is not None:
        config.alpamayo_root = alpamayo_root
    if a2d2_root is not None:
        config.a2d2_root = a2d2_root
    config.allow_stub = allow_stub
    with open(CONFIG_PATH, "w") as f:
        json.dump(config.model_dump(), f, indent=2)

    return result


@app.get("/api/dataset/ingest/status")
def ingest_status(sequence_id: str = "seq_001"):
    from sensorflow.dataset_fusion_engine import DatasetFusionEngine
    engine = DatasetFusionEngine()
    status = engine.get_status(sequence_id)
    manifest = Path("runs/pipeline") / sequence_id / "manifest.json"
    if manifest.exists():
        with open(manifest) as f:
            manifest_data = json.load(f)
        status["manifest_exists"] = True
        status["frames"] = len(manifest_data.get("frames", []))
        status["vendor"] = manifest_data.get("vendor", "unknown")
        tax = manifest_data.get("taxonomy_manifest") or {}
        status["demo_stub"] = bool(tax.get("demo_stub", status.get("demo_stub", False)))
        status["stub_note"] = tax.get("stub_note")
    return {"status": "ok", "sequence_id": sequence_id, **status}


def _auto_label_next_steps(
    *,
    expected_frames: int,
    checkpoint_exists: bool,
    no_sam: bool,
    sequence_id: str,
) -> List[str]:
    steps: List[str] = []
    if expected_frames == 0:
        steps.append(
            f"Run Step 2 (3D Ingest) or Load all datasets — pick sequence "
            f"(e.g. {sequence_id}_local) with frames > 0"
        )
    if not no_sam and not checkpoint_exists:
        steps.append(SAM_CHECKPOINT_HINT)
        steps.append("Or enable “Skip SAM” for GT/synthetic fallback only (not a model success)")
    if expected_frames > 0 and not steps:
        steps.append("Proceed to Temporal Tracking when proposals look correct")
    return steps


@app.post("/api/perception/auto-label")
def auto_label(params: AutoLabelParams):
    from sensorflow.perception_automator import PerceptionAutomator
    from sensorflow.schemas.unified_frame import UnifiedSequence
    from sensorflow.execution_ledger import create_execution, mark_running, finalize
    from sensorflow.execution_ops import artifact_info

    manifest = Path("runs/pipeline") / params.sequence_id / "manifest.json"
    ckpt = Path(params.sam_checkpoint)
    record = create_execution(
        "auto_label_3d",
        configuration_snapshot=params.model_dump(),
        input_artifacts=[str(manifest), params.sam_checkpoint],
    )
    exec_id = record["execution_id"]

    def _fail(
        message: str,
        *,
        errors: Optional[List[str]] = None,
        http_status: int = 400,
        expected_frames: int = 0,
        next_steps: Optional[List[str]] = None,
    ):
        steps = next_steps or _auto_label_next_steps(
            expected_frames=expected_frames,
            checkpoint_exists=ckpt.exists(),
            no_sam=params.no_sam,
            sequence_id=params.sequence_id,
        )
        final = finalize(
            exec_id,
            "FAILED",
            records_discovered=expected_frames,
            records_processed=0,
            metrics={
                "model": "none (no_sam)" if params.no_sam else "sam_vit_b",
                "checkpoint": artifact_info(ckpt),
                "sam_ran": False,
                "no_sam": params.no_sam,
                "frames_expected": expected_frames,
                "frames_processed": 0,
                "predictions_generated": 0,
                "inference_calls": 0,
            },
            errors=errors or [message],
            warnings=[],
            process_invoked=False,
            outputs_valid=False,
        )
        payload = {
            "status": "FAILED",
            "execution_id": exec_id,
            "verified": final.get("verified"),
            "duration_ms": final.get("duration_ms"),
            "message": message,
            "frames_expected": expected_frames,
            "frames_processed": 0,
            "predictions_generated": 0,
            "checkpoint": artifact_info(ckpt),
            "sam_ran": False,
            "demo_stub": None,
            "next_steps": steps,
            "events": final.get("events"),
        }
        raise HTTPException(status_code=http_status, detail=payload)

    if not manifest.exists():
        _fail(
            "Manifest not found. Run 3D Ingest (Step 2) first.",
            errors=["Manifest not found. Run ingest first."],
        )

    sequence = UnifiedSequence.load(manifest)
    expected_frames = len(sequence.frames)
    manifest_frame_ids = {f.frame_id for f in sequence.frames}
    demo_stub = bool(sequence.taxonomy_manifest.get("demo_stub"))

    if expected_frames == 0:
        _fail(
            f"Sequence '{params.sequence_id}' has 0 ingested frames. "
            "Run 3D Ingest or Load all datasets before auto-label.",
            expected_frames=0,
        )

    if not params.no_sam and not ckpt.exists():
        _fail(
            f"SAM checkpoint not found at {params.sam_checkpoint}. "
            "Download weights or enable “Skip SAM” for demo/GT fallback only.",
            expected_frames=expected_frames,
        )

    sam_ran = False
    model_name = "sam_vit_b" if not params.no_sam else "none (no_sam)"
    warnings: List[str] = []
    if params.no_sam:
        warnings.append("no_sam=true — SAM not invoked; proposals use GT/synthetic fallback only")

    mark_running(exec_id)
    output_dir = manifest.parent / "proposals"
    if output_dir.exists():
        for stale in output_dir.glob("*.json"):
            if stale.stem not in manifest_frame_ids:
                stale.unlink(missing_ok=True)

    try:
        automator = PerceptionAutomator(
            sam_checkpoint=params.sam_checkpoint,
            device=params.device,
            use_sam=not params.no_sam,
        )
        automator.run_sequence(sequence, output_dir)
        sam_ran = bool(
            not params.no_sam
            and getattr(automator, "use_sam", False)
            and automator._mask_generator is not None
        )
    except FileNotFoundError as e:
        _fail(
            str(e),
            errors=[str(e)],
            expected_frames=expected_frames,
        )
    except Exception as e:
        finalize(
            exec_id,
            "FAILED",
            errors=[str(e)],
            process_invoked=False,
            outputs_valid=False,
        )
        raise HTTPException(
            status_code=500,
            detail={
                "status": "FAILED",
                "message": f"Auto-label failed: {str(e)}",
                "execution_id": exec_id,
            },
        )

    proposals_dir = manifest.parent / "proposals"
    proposal_files = (
        [pf for pf in proposals_dir.glob("*.json") if pf.stem in manifest_frame_ids]
        if proposals_dir.exists()
        else []
    )
    num_frames = len(proposal_files)
    pred_count = 0
    for pf in proposal_files:
        try:
            pdata = json.loads(pf.read_text())
            if isinstance(pdata, list):
                pred_count += len(pdata)
            elif isinstance(pdata, dict):
                pred_count += len(pdata.get("proposals") or pdata.get("objects") or [])
        except Exception:
            pass

    if params.no_sam:
        status = "NOT_EXECUTED" if num_frames > 0 else "FAILED"
        warnings.append("Model not run (no_sam); do not treat as SAM SUCCESS")
    elif not sam_ran:
        status = "FAILED"
        warnings.append("SAM did not run — no valid proposals produced")
    elif num_frames == 0:
        status = "FAILED"
    elif num_frames < expected_frames:
        status = "PARTIAL_SUCCESS"
    else:
        status = "SUCCEEDED"

    next_steps = _auto_label_next_steps(
        expected_frames=expected_frames,
        checkpoint_exists=ckpt.exists(),
        no_sam=params.no_sam,
        sequence_id=params.sequence_id,
    )
    if status == "SUCCEEDED":
        message = (
            f"SAM auto-label: processed {num_frames}/{expected_frames} frames, "
            f"{pred_count} predictions → {proposals_dir}"
        )
    elif status == "NOT_EXECUTED":
        message = (
            f"GT/synthetic fallback (SAM skipped): {num_frames}/{expected_frames} frames, "
            f"{pred_count} predictions → {proposals_dir}"
        )
    elif status == "PARTIAL_SUCCESS":
        message = (
            f"Partial SAM run: {num_frames}/{expected_frames} frames, "
            f"{pred_count} predictions → {proposals_dir}"
        )
    else:
        message = (
            f"Auto-label failed: 0/{expected_frames} frames produced. "
            f"Check ingest, SAM checkpoint, and sequence selection."
        )

    final = finalize(
        exec_id,
        status,
        records_discovered=expected_frames,
        records_processed=num_frames,
        records_succeeded=num_frames,
        records_failed=max(0, expected_frames - num_frames),
        metrics={
            "model": model_name,
            "checkpoint": artifact_info(ckpt),
            "sam_ran": sam_ran,
            "no_sam": params.no_sam,
            "frames_expected": expected_frames,
            "frames_processed": num_frames,
            "predictions_generated": pred_count,
            "inference_calls": num_frames if sam_ran else 0,
            "output_dir": str(proposals_dir),
            "demo_stub": demo_stub,
        },
        output_artifacts=[{"path": str(proposals_dir), "kind": "proposals_dir", "files": num_frames}],
        warnings=warnings,
        process_invoked=sam_ran or params.no_sam,
        outputs_valid=num_frames > 0,
    )

    return {
        "status": final["status"],
        "execution_id": final["execution_id"],
        "verified": final.get("verified"),
        "duration_ms": final.get("duration_ms"),
        "proposals_dir": str(proposals_dir),
        "frames_processed": num_frames,
        "frames_expected": expected_frames,
        "predictions_generated": pred_count,
        "inference_calls": num_frames if sam_ran else 0,
        "model": model_name,
        "checkpoint": artifact_info(ckpt),
        "sam_ran": sam_ran,
        "demo_stub": demo_stub,
        "message": message,
        "next_steps": next_steps,
        "events": final.get("events"),
    }


@app.post("/api/perception/track")
def run_tracking(params: TrackParams):
    from sensorflow.temporal_tracker import TemporalTracker

    proposals_dir = Path("runs/pipeline") / params.sequence_id / "proposals"
    if not proposals_dir.exists():
        raise HTTPException(status_code=400, detail="Proposals not found. Run auto-label first.")

    output = Path("runs/pipeline") / params.sequence_id / "tracks.json"
    try:
        tracker = TemporalTracker()
        proposals = TemporalTracker.load_proposals(proposals_dir)
        tracks = tracker.run_sequence(proposals, output)
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Tracking failed: {str(e)}")

    return {"status": "ok", "tracks_file": str(output), "num_tracks": len(tracks)}


@app.post("/api/benchmark")
def run_benchmark(params: BenchmarkParams):
    from sensorflow.quality_gate import QualityGate
    from sensorflow.schemas.unified_frame import UnifiedSequence

    manifest = Path("runs/pipeline") / params.sequence_id / "manifest.json"
    tracks_path = Path("runs/pipeline") / params.sequence_id / "tracks.json"
    if not manifest.exists():
        raise HTTPException(status_code=400, detail="Manifest not found.")

    try:
        sequence = UnifiedSequence.load(manifest)
        pred_tracks = json.loads(tracks_path.read_text()) if tracks_path.exists() else []
        gate = QualityGate()
        results = gate.evaluate(sequence, pred_tracks)
        gate.save_results(params.sequence_id, results)
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Benchmark failed: {str(e)}")

    return {"status": "ok", "metrics": results["metric_card"]}


@app.post("/api/gates/quality")
def quality_gate_eval(params: GateParams):
    from sensorflow.quality_gate import QualityGate
    from sensorflow.mitl_copilot import MitlCopilot
    from sensorflow.schemas.unified_frame import UnifiedSequence

    manifest = Path("runs/pipeline") / params.sequence_id / "manifest.json"
    tracks_path = Path("runs/pipeline") / params.sequence_id / "tracks.json"
    if not manifest.exists() or not tracks_path.exists():
        raise HTTPException(status_code=400, detail="Run ingest, auto-label, and track first.")

    try:
        sequence = UnifiedSequence.load(manifest)
        with open(tracks_path) as f:
            pred_tracks = json.load(f)
        gate = QualityGate()
        results = gate.evaluate(sequence, pred_tracks)
        gate.save_results(params.sequence_id, results)
        if not results["passed"]:
            copilot = MitlCopilot()
            copilot.route_edge_cases(params.sequence_id, results["metric_card"], pred_tracks)
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Quality gate failed: {str(e)}")

    report = results["quality_report"]
    return {"status": "ok", "metric_card": results["metric_card"], **report}


@app.post("/api/gates/launch")
def launch_gate_eval(params: GateParams):
    from sensorflow.launch_gate_evaluator import LaunchGateEvaluator
    config = load_config()
    evaluator = LaunchGateEvaluator(Path(config.gate_thresholds_path))
    result = evaluator.evaluate(params.sequence_id)
    return {"status": "ok", **result}


@app.get("/api/pipeline/status")
def pipeline_status(sequence_id: str = "seq_001"):
    state_path = Path("runs/pipeline/state.json")
    state = {}
    if state_path.exists():
        with open(state_path) as f:
            state = json.load(f)

    seq_state = state.get(sequence_id, {})
    base = Path("runs/pipeline") / sequence_id
    manifest = base / "manifest.json"
    proposals_dir = base / "proposals"
    tracks = base / "tracks.json"
    benchmark = base / "benchmark" / "metric_card.json"
    quality_report = base / "benchmark" / "quality_report.json"

    ingest_complete = manifest.exists()
    perception_complete = proposals_dir.exists() and any(proposals_dir.glob("*.json"))
    tracking_complete = tracks.exists()
    benchmark_complete = benchmark.exists()

    frames_ingested = None
    frames_processed = None
    demo_stub = None
    if ingest_complete:
        try:
            with open(manifest) as f:
                manifest_data = json.load(f)
            frames_ingested = len(manifest_data.get("frames", []))
            demo_stub = bool((manifest_data.get("taxonomy_manifest") or {}).get("demo_stub"))
        except Exception:
            frames_ingested = seq_state.get("frames")
            demo_stub = seq_state.get("demo_stub")
    if perception_complete:
        frames_processed = len(list(proposals_dir.glob("*.json")))

    # Completion stages: True when done, None when not run (UI must not show FAIL).
    # Launch gate: True/False only after evaluation; None beforehand.
    launch_gate_passed = seq_state.get("launch_gate_passed")
    if "launch_gate_passed" not in seq_state:
        launch_gate_passed = None

    quality_passed = None
    if quality_report.exists():
        try:
            with open(quality_report) as f:
                quality_passed = bool(json.load(f).get("passed"))
        except Exception:
            quality_passed = benchmark_complete

    return {
        "status": "ok",
        "sequence_id": sequence_id,
        "active_sequence_id": resolve_active_sequence_id(sequence_id),
        "ingest_complete": True if ingest_complete else None,
        "perception_complete": True if perception_complete else None,
        "tracking_complete": True if tracking_complete else None,
        "benchmark_complete": True if benchmark_complete else None,
        "quality_gate_passed": quality_passed,
        "launch_gate_passed": launch_gate_passed,
        "frames_ingested": frames_ingested,
        "frames_processed": frames_processed,
        "demo_stub": demo_stub,
        "state": seq_state,
    }


@app.get("/api/pipeline/sequences")
def pipeline_sequences():
    from sensorflow import pipeline_artifacts as artifacts
    return {"status": "ok", "sequences": artifacts.list_sequences()}


@app.get("/api/pipeline/artifacts")
def pipeline_artifacts(sequence_id: str = "seq_001"):
    from sensorflow import pipeline_artifacts as artifacts
    return {"status": "ok", **artifacts.artifacts_summary(sequence_id)}


@app.get("/api/pipeline/frames")
def pipeline_frames(sequence_id: str = "seq_001"):
    from sensorflow import pipeline_artifacts as artifacts
    return artifacts.list_frames(sequence_id)


@app.get("/api/pipeline/frame")
def pipeline_frame(sequence_id: str, frame_id: str):
    from sensorflow import pipeline_artifacts as artifacts
    try:
        return artifacts.get_frame(sequence_id, frame_id)
    except FileNotFoundError as e:
        raise HTTPException(status_code=404, detail=str(e))


@app.get("/api/pipeline/file")
def pipeline_file(path: str):
    """Serve a local run/data artifact with path-traversal protection."""
    from sensorflow import pipeline_artifacts as artifacts
    try:
        file_path, media_type = artifacts.resolve_file(path)
    except PermissionError as e:
        raise HTTPException(status_code=403, detail=str(e))
    except FileNotFoundError as e:
        raise HTTPException(status_code=404, detail=str(e))
    return FileResponse(file_path, media_type=media_type)


@app.get("/api/dataset/browse")
def dataset_browse(
    source_path: str = "data",
    limit: int = 48,
    dataset_type: Optional[str] = None,
):
    """Validate Images Path by listing browsable local driving-relevant frames."""
    from sensorflow import pipeline_artifacts as artifacts

    dtype = dataset_type or load_config().dataset_type
    return artifacts.scan_source_images(source_path, limit=limit, dataset_type=dtype)


@app.get("/api/mitl/queue")
def get_mitl_queue(sequence_id: str = "seq_001"):
    from sensorflow.mitl_copilot import MitlCopilot
    copilot = MitlCopilot()
    return {"status": "ok", "queue": copilot.get_queue(sequence_id)}


# L4 Perception Label Evaluation platform routes (must precede the static mount).
from sensorflow.evaluation.api import router as labeleval_router
app.include_router(labeleval_router)

# Aggregate-first mega-scale evaluation layer (metric cube, async runs, query API).
from sensorflow.megaeval.api import router as megaeval_router
app.include_router(megaeval_router)

# Sequential regression detection engine (anytime-valid, budgeted, paired).
from sensorflow.seqeval.api import router as seqeval_router
app.include_router(seqeval_router)

# BEV-Fusion perception engine (camera+LiDAR fusion, masklet tracking, self-eval).
from sensorflow.bevfusion.api import router as bevfusion_router
app.include_router(bevfusion_router)

# Safety & compliance layer (ODD coverage, release gates, SSAM, calibration,
# discrepancy mining, scenario DB, semantic mining).
from sensorflow.safety.api import router as safety_router
app.include_router(safety_router)

# Regression Root Cause Analysis workbench (staged offline-vs-shadow forensics).
from sensorflow.rca.api import router as rca_router
app.include_router(rca_router)

# Hill Climbing EM: adaptive EM development & interview-readiness platform.
from sensorflow.hillclimb.api import router as hillclimb_router
app.include_router(hillclimb_router)

# Multimodal rare-event mining & perception QA (costumed-pedestrian miner).
from sensorflow.raremine.api import router as raremine_router
app.include_router(raremine_router)

# Vitis Vision acceleration layer (HIL quantization gap, accelerated ISP +
# synthetic edge cases, temporal/stereo stability). Emulated backend; no FPGA.
from sensorflow.vitis.api import router as vitis_router
app.include_router(vitis_router)

# Agentic launch readiness & misclassification triage (five-layer pipeline,
# deterministic policy engine, evidence graphs, evaluation flywheel).
from sensorflow.agentic.api import router as agentic_router
app.include_router(agentic_router)

# Next-gen AV perception evaluation: counterfactual simulation + validity
# gating, closed-loop behavioral evaluation, safety-informed metrics,
# compute dedup + launch-eval gauntlet scheduling.
from sensorflow.nextgen.api import router as nextgen_router
app.include_router(nextgen_router)

# Agentic Retrospective Safety Analyzer (evidence-tiered failure retrospectives,
# safety-case RAG, deterministic severity + launch policy gate).
from sensorflow.retro.api import router as retro_router
app.include_router(retro_router)

# Studio UX support (dashboard layout persistence + BEV frame replay for the
# interactive canvas). Read-only over other packages' data.
from sensorflow.studio_ux.api import router as studio_ux_router
app.include_router(studio_ux_router)

# Production hardening layer (audit browser, readiness scorecard, funnel,
# sampling/quality/HITL demos on seeded synthetic fixtures).
from sensorflow.hardening.api import router as hardening_router
app.include_router(hardening_router)

# Studio 2.0 control plane: unified entity registry, deterministic release
# gate composing safety/seqeval/megaeval (+ agentic/nextgen when available),
# hardware-aware gate matrix, observability funnel, closed-loop demo.
from sensorflow.studio2.api import router as studio2_router
app.include_router(studio2_router)

# ROTR: right-of-the-road violation detection, causal-layer attribution,
# counterfactual consequence, taxonomy mining, governed flywheel + stop-ship.
from sensorflow.rotr.api import router as rotr_router
app.include_router(rotr_router)

# In-app help chatbot (FAQ / page-guide matcher; optional Ollama enrichment).
from sensorflow.help.api import router as help_router
app.include_router(help_router)

# Product version + About catalog (GET /api/about, GET /api/version).
from sensorflow.about.api import router as about_router
app.include_router(about_router)

# Mount static folder
static_dir = Path(__file__).parent / "static"
if static_dir.exists():
    app.mount("/", StaticFiles(directory=str(static_dir), html=True), name="static")

if __name__ == "__main__":
    import uvicorn
    uvicorn.run("app_backend:app", host="127.0.0.1", port=8000, reload=True)