diff --git a/README.md b/README.md index fc97ecbdf14a72fe720138f5e395d0b946a57008..7203845275064ae4e85fb70b47475d0985a2537c 100644 --- a/README.md +++ b/README.md @@ -29,3 +29,9 @@ copilot — are not functional here; the dashboards and analytics APIs are. Deployed from the project repository via `deploy/huggingface/deploy.py` (uploads files with `huggingface_hub`; the frontend is built inside the Docker image). + +## Version + +Current version is shown in the Studio AppBar (`vX.Y.Z` chip) and Help → About, +with release notes for each notable version. The same catalog is served at +`GET /api/about` and `GET /api/version`. See `CHANGELOG.md` in the GitHub repo. diff --git a/app_backend.py b/app_backend.py index 742c2211e4408a60f5e58abf27988f40a971f6ef..ff2bdbacc81f1bb690109ffd56d19d4c5543b37f 100644 --- a/app_backend.py +++ b/app_backend.py @@ -5,21 +5,24 @@ import json import subprocess import signal from pathlib import Path -from typing import List, Optional +from typing import List, Optional, Any, Dict from pydantic import BaseModel import httpx -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Query from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, PlainTextResponse app = FastAPI(title="Sensorflow Studio Backend") # State tracking ACTIVE_TRAIN_PROC: Optional[subprocess.Popen] = None +ACTIVE_TRAIN_EXECUTION_ID: Optional[str] = None TRAINING_LOGS_ACCUMULATED = "" TRAINING_LOSSES: List[float] = [] TOTAL_EPOCHS = 10 CURRENT_EPOCH = 0 +LAST_TRAIN_EXIT_CODE: Optional[int] = None +LAST_TRAIN_COMMAND: Optional[List[str]] = None # Configuration CONFIG_PATH = Path("runs/studio_config.json") @@ -32,7 +35,7 @@ class StudioConfig(BaseModel): model_type: str = "yolov8" pipeline_mode: str = "3d" sam_checkpoint: str = "models/sam_vit_b.pth" - vendors: List[str] = ["alpamayo", "waymo"] + vendors: List[str] = ["alpamayo", "waymo", "a2d2"] gate_thresholds_path: str = "runs/pipeline/gate_thresholds.json" sequence_id: str = "seq_001" @@ -105,32 +108,236 @@ def save_yaml_content(req: SaveYamlRequest): @app.get("/api/precheck") def precheck(): - # Verify scripts are present - train_exists = Path("train.py").exists() - infer_exists = Path("infer.py").exists() - grader_exists = Path("autograder.py").exists() + """Script verification — existence alone is NOT success.""" + from sensorflow.execution_ops import verify_script + + reports = { + "train.py": verify_script("train.py"), + "infer.py": verify_script("infer.py"), + "autograder.py": verify_script("autograder.py"), + } + all_ok = all( + r.get("exists") and r.get("syntax_valid") and r.get("dry_run_ok") + for r in reports.values() + ) + any_missing = any(not r.get("exists") for r in reports.values()) + return { + "status": "ok" if all_ok else ("warning" if not any_missing else "failed"), + "message": ( + "Scripts exist, syntax-valid, and --help dry-run succeeded." + if all_ok + else "Script verification incomplete — see reports (file Found alone is not enough)." + ), + "scripts": reports, + "verified": all_ok, + } + + +@app.get("/api/health") +def health(): + from sensorflow.execution_ops import backend_health + from sensorflow.execution_ledger import get_strict_mode + + payload = backend_health() + payload["strict_mode"] = get_strict_mode() + payload["backend_connected"] = True + return payload + + +@app.get("/api/strict-mode") +def get_strict_mode_api(): + from sensorflow.execution_ledger import get_strict_mode + + return {"enabled": get_strict_mode()} + + +@app.post("/api/strict-mode") +def set_strict_mode_api(params: dict): + from sensorflow.execution_ledger import set_strict_mode + + enabled = bool(params.get("enabled", False)) + return set_strict_mode(enabled) + + +@app.get("/api/executions") +def list_executions_api( + operation: Optional[str] = None, + status: Optional[str] = None, + limit: int = Query(50, ge=1, le=500), +): + from sensorflow.execution_ledger import list_executions + + return {"status": "ok", "executions": list_executions(operation=operation, status=status, limit=limit)} + + +@app.get("/api/executions/{execution_id}") +def get_execution_api(execution_id: str): + from sensorflow.execution_ledger import load_execution + + try: + return load_execution(execution_id) + except FileNotFoundError: + raise HTTPException(status_code=404, detail=f"Execution not found: {execution_id}") + + +@app.get("/api/executions/{execution_id}/log") +def get_execution_log_api(execution_id: str): + from sensorflow.execution_ledger import get_log_text, load_execution + + try: + load_execution(execution_id) + except FileNotFoundError: + raise HTTPException(status_code=404, detail=f"Execution not found: {execution_id}") + return PlainTextResponse(get_log_text(execution_id) or "(no log lines)") + + +@app.post("/api/yaml/validate") +def validate_yaml_api(params: dict): + from sensorflow.execution_ops import validate_yaml_semantics + from sensorflow.execution_ledger import create_execution, mark_running, finalize + + path = params.get("path") or params.get("yaml_path") or "coco8.yaml" + record = create_execution( + "yaml_validate", + configuration_snapshot={"yaml_path": path}, + input_artifacts=[path], + ) + mark_running(record["execution_id"]) + report = validate_yaml_semantics(path) + final = finalize( + record["execution_id"], + report.get("status") or "FAILED", + metrics={ + "class_count": report.get("class_count"), + "image_counts": report.get("image_counts"), + "label_counts": report.get("label_counts"), + }, + errors=report.get("errors"), + warnings=report.get("warnings"), + output_artifacts=[{"path": path, "hash": report.get("content_hash")}], + process_invoked=False, + outputs_valid=report.get("status") == "SUCCEEDED", + ) + report["execution_id"] = final["execution_id"] + report["execution"] = { + "execution_id": final["execution_id"], + "status": final["status"], + "verified": final.get("verified"), + "duration_ms": final.get("duration_ms"), + } + return report + + +@app.get("/api/scripts/verify") +def scripts_verify_api(): + from sensorflow.execution_ops import verify_script - if not (train_exists and infer_exists and grader_exists): - return { - "status": "warning", - "message": "Some scripts are missing in the current folder. Please verify train.py, infer.py, and autograder.py exist." - } return { - "status": "success", - "message": "All workspace training and inference scripts are verified." + "status": "ok", + "scripts": { + "train.py": verify_script("train.py"), + "infer.py": verify_script("infer.py"), + "autograder.py": verify_script("autograder.py"), + }, } + +@app.post("/api/dataset/load") +def dataset_load(params: dict): + """Real Load & Preprocess with discovery evidence (not catalog KPIs).""" + from sensorflow.execution_ops import load_and_preprocess + from sensorflow.execution_ledger import create_execution, mark_running, finalize + + source_path = params.get("source_path") or load_config().source_path + yaml_path = params.get("yaml_path") or load_config().yaml_path + dataset_type = params.get("dataset_type") or load_config().dataset_type + + record = create_execution( + "dataset_load", + configuration_snapshot={ + "source_path": source_path, + "yaml_path": yaml_path, + "dataset_type": dataset_type, + }, + input_artifacts=[source_path, yaml_path], + ) + mark_running(record["execution_id"]) + result = load_and_preprocess(source_path, yaml_path=yaml_path, dataset_type=dataset_type) + disc = result["discovery"] + final = finalize( + record["execution_id"], + result["status"], + records_discovered=disc.get("images_discovered", 0), + records_processed=disc.get("images_discovered", 0), + records_succeeded=disc.get("images_readable", 0), + records_failed=disc.get("images_corrupt", 0), + metrics=result["metrics"], + errors=disc.get("errors"), + warnings=disc.get("warnings"), + output_artifacts=[{"path": result["manifest_path"], "kind": "load_manifest"}], + process_invoked=False, + outputs_valid=disc.get("images_readable", 0) > 0 and result["status"] in ( + "SUCCEEDED", + "PARTIAL_SUCCESS", + ), + ) + return { + "status": final["status"], + "execution_id": final["execution_id"], + "verified": final.get("verified"), + "duration_ms": final.get("duration_ms"), + "message": ( + f"Loaded {disc.get('images_readable', 0)}/{disc.get('images_discovered', 0)} images " + f"from {source_path}" + if disc.get("images_readable") + else f"FAILED: 0 images loaded from {source_path}" + ), + "discovery": disc, + "metrics": result["metrics"], + "yaml_validation": result.get("yaml_validation"), + "reconciliation": result.get("reconciliation"), + "manifest_path": result.get("manifest_path"), + "catalog_only": False, + "browsable": bool(result["metrics"].get("browsable")), + "events": final.get("events"), + } + + @app.post("/api/train/start") def start_train(params: TrainParams): - global ACTIVE_TRAIN_PROC, TRAINING_LOGS_ACCUMULATED, TRAINING_LOSSES, TOTAL_EPOCHS, CURRENT_EPOCH + global ACTIVE_TRAIN_PROC, ACTIVE_TRAIN_EXECUTION_ID, TRAINING_LOGS_ACCUMULATED + global TRAINING_LOSSES, TOTAL_EPOCHS, CURRENT_EPOCH, LAST_TRAIN_EXIT_CODE, LAST_TRAIN_COMMAND + from sensorflow.execution_ledger import create_execution, mark_running, finalize + if ACTIVE_TRAIN_PROC and ACTIVE_TRAIN_PROC.poll() is None: raise HTTPException(status_code=400, detail="Training is already running.") - # Reset logs & stats + weights_path = Path(params.model) + # Allow ultralytics named weights (e.g. yolov8n.pt) that may download; still record path + yaml_path = Path(params.data) + if not yaml_path.exists(): + record = create_execution( + "training", + configuration_snapshot=params.dict(), + input_artifacts=[params.data, params.model], + ) + finalize( + record["execution_id"], + "FAILED", + errors=[f"Dataset YAML not found: {params.data}"], + process_invoked=False, + outputs_valid=False, + ) + raise HTTPException( + status_code=400, + detail={"message": f"Dataset YAML not found: {params.data}", "execution_id": record["execution_id"]}, + ) + TRAINING_LOGS_ACCUMULATED = f"Starting YOLO training with base model {params.model}...\n" TRAINING_LOSSES = [] TOTAL_EPOCHS = params.epochs CURRENT_EPOCH = 0 + LAST_TRAIN_EXIT_CODE = None cmd = [ sys.executable, "train.py", @@ -138,8 +345,17 @@ def start_train(params: TrainParams): "--batch", str(params.batch), "--device", params.device, "--model", params.model, - "--data", params.data + "--data", params.data, ] + LAST_TRAIN_COMMAND = cmd + + record = create_execution( + "training", + configuration_snapshot=params.dict(), + input_artifacts=[params.data, params.model], + command=cmd, + ) + ACTIVE_TRAIN_EXECUTION_ID = record["execution_id"] try: ACTIVE_TRAIN_PROC = subprocess.Popen( @@ -148,47 +364,80 @@ def start_train(params: TrainParams): stderr=subprocess.STDOUT, text=True, bufsize=1, - preexec_fn=os.setsid if os.name != 'nt' else None + preexec_fn=os.setsid if os.name != "nt" else None, ) except Exception as e: + finalize( + record["execution_id"], + "FAILED", + errors=[f"Failed to spawn training process: {e}"], + process_invoked=False, + outputs_valid=False, + ) + ACTIVE_TRAIN_EXECUTION_ID = None raise HTTPException(status_code=500, detail=f"Failed to spawn training process: {str(e)}") - return {"status": "started"} + mark_running(record["execution_id"], process_id=ACTIVE_TRAIN_PROC.pid) + return { + "status": "started", + "execution_id": record["execution_id"], + "process_id": ACTIVE_TRAIN_PROC.pid, + "command": cmd, + } + @app.post("/api/train/stop") def stop_train(): - global ACTIVE_TRAIN_PROC + global ACTIVE_TRAIN_PROC, ACTIVE_TRAIN_EXECUTION_ID, LAST_TRAIN_EXIT_CODE + from sensorflow.execution_ledger import finalize, append_log + if not ACTIVE_TRAIN_PROC or ACTIVE_TRAIN_PROC.poll() is not None: - return {"status": "not_running"} + return {"status": "not_running", "execution_id": ACTIVE_TRAIN_EXECUTION_ID} + exec_id = ACTIVE_TRAIN_EXECUTION_ID try: - if os.name != 'nt': + if os.name != "nt": os.killpg(os.getpgid(ACTIVE_TRAIN_PROC.pid), signal.SIGTERM) else: ACTIVE_TRAIN_PROC.terminate() ACTIVE_TRAIN_PROC.wait(timeout=3) - except Exception as e: - # Fallback to simple kill + except Exception: try: ACTIVE_TRAIN_PROC.kill() except Exception: pass - return {"status": "stopped"} + LAST_TRAIN_EXIT_CODE = ACTIVE_TRAIN_PROC.poll() + if exec_id: + append_log(exec_id, "Training terminated by user") + finalize( + exec_id, + "CANCELLED", + exit_code=LAST_TRAIN_EXIT_CODE, + process_invoked=True, + outputs_valid=False, + errors=["Cancelled by user"], + ) + ACTIVE_TRAIN_PROC = None + return {"status": "stopped", "execution_id": exec_id, "exit_code": LAST_TRAIN_EXIT_CODE} + @app.get("/api/train/status") def get_train_status(): - global ACTIVE_TRAIN_PROC, TRAINING_LOGS_ACCUMULATED, TRAINING_LOSSES, CURRENT_EPOCH, TOTAL_EPOCHS + global ACTIVE_TRAIN_PROC, ACTIVE_TRAIN_EXECUTION_ID, TRAINING_LOGS_ACCUMULATED + global TRAINING_LOSSES, CURRENT_EPOCH, TOTAL_EPOCHS, LAST_TRAIN_EXIT_CODE, LAST_TRAIN_COMMAND + from sensorflow.execution_ledger import append_log, finalize, sha256_file + from sensorflow.execution_ops import artifact_info + running = False - + just_finished = False + exit_code = LAST_TRAIN_EXIT_CODE + if ACTIVE_TRAIN_PROC: - # Read new output line by line (non-blocking) - # Use select or set standard output stream non-blocking if needed, or read from stdout. - # To avoid blocking, we read line-by-line if there is data. import select + if ACTIVE_TRAIN_PROC.poll() is None: running = True - # Read available outputs while True: r, _, _ = select.select([ACTIVE_TRAIN_PROC.stdout], [], [], 0.02) if ACTIVE_TRAIN_PROC.stdout in r: @@ -196,23 +445,22 @@ def get_train_status(): if not line: break TRAINING_LOGS_ACCUMULATED += line - # Parse epoch loss from Ultralytics logs if visible - # e.g., "1/10 3.5G 1.234" - # Ultralytics log format standard contains epoch index and training loss + if ACTIVE_TRAIN_EXECUTION_ID: + try: + append_log(ACTIVE_TRAIN_EXECUTION_ID, line.rstrip()) + except Exception: + pass if "epoch" in line.lower() or "/" in line: try: - # Try to extract numbers parts = line.split() if len(parts) >= 3 and "/" in parts[0]: ep_part = parts[0].split("/") epoch_num = int(ep_part[0]) CURRENT_EPOCH = epoch_num - # Try to grab a float that might be loss for p in parts[2:]: try: val = float(p) if 0.0 < val < 10.0: - # Found a candidate loss value if len(TRAINING_LOSSES) < epoch_num: TRAINING_LOSSES.append(val) else: @@ -225,54 +473,237 @@ def get_train_status(): else: break else: - # Done running, read all remaining output remaining = ACTIVE_TRAIN_PROC.stdout.read() if remaining: TRAINING_LOGS_ACCUMULATED += remaining + exit_code = ACTIVE_TRAIN_PROC.poll() + LAST_TRAIN_EXIT_CODE = exit_code + just_finished = True ACTIVE_TRAIN_PROC = None - # Calculate mock losses if empty to display curves in frontend - if not running and not TRAINING_LOSSES and TRAINING_LOGS_ACCUMULATED: - # Finished but no parsed losses, generate dummy decreasing loss - TRAINING_LOSSES = [2.5 - (2.0 / TOTAL_EPOCHS) * i for i in range(TOTAL_EPOCHS)] - progress = 0.0 - if TOTAL_EPOCHS > 0: - progress = float(CURRENT_EPOCH) / float(TOTAL_EPOCHS) - if not running and progress == 0.0: - progress = 1.0 + if TOTAL_EPOCHS > 0 and CURRENT_EPOCH > 0: + progress = min(1.0, float(CURRENT_EPOCH) / float(TOTAL_EPOCHS)) + elif not running and exit_code == 0 and CURRENT_EPOCH > 0: + progress = 1.0 + # Never invent progress=1.0 on failure or empty parse + + checkpoint = Path("runs/detect/coco_finetuned/weights/best.pt") + ckpt_info = artifact_info(checkpoint) if checkpoint.exists() else {"path": str(checkpoint), "exists": False} + + if just_finished and ACTIVE_TRAIN_EXECUTION_ID: + status = "SUCCEEDED" if exit_code == 0 else "FAILED" + if exit_code is None: + status = "FAILED" + finalize( + ACTIVE_TRAIN_EXECUTION_ID, + status, + exit_code=exit_code, + metrics={ + "epochs_requested": TOTAL_EPOCHS, + "epochs_observed": CURRENT_EPOCH, + "losses_parsed": len(TRAINING_LOSSES), + "losses": TRAINING_LOSSES, + "checkpoint": ckpt_info, + }, + output_artifacts=[ckpt_info] if ckpt_info.get("exists") else [], + process_invoked=True, + outputs_valid=bool(ckpt_info.get("exists")) and exit_code == 0, + errors=[] if exit_code == 0 else [f"Training exit_code={exit_code}"], + ) return { "running": running, "progress": progress, - "logs": TRAINING_LOGS_ACCUMULATED[-20000:], # Limit log size sent to browser - "losses": TRAINING_LOSSES + "logs": TRAINING_LOGS_ACCUMULATED[-20000:], + "losses": TRAINING_LOSSES, # only real parsed losses — never fabricated + "losses_fabricated": False, + "exit_code": exit_code, + "process_id": None if not running else (ACTIVE_TRAIN_PROC.pid if ACTIVE_TRAIN_PROC else None), + "command": LAST_TRAIN_COMMAND, + "execution_id": ACTIVE_TRAIN_EXECUTION_ID, + "epochs_observed": CURRENT_EPOCH, + "epochs_requested": TOTAL_EPOCHS, + "checkpoint": ckpt_info, + "status": ( + "RUNNING" if running else ( + "SUCCEEDED" if exit_code == 0 else ( + "FAILED" if exit_code not in (None, 0) else "NOT_EXECUTED" + ) + ) + ), } + @app.post("/api/infer/run") def run_infer(params: InferParams): - # Run the infer.py script using subprocess - cmd = [ - sys.executable, "infer.py", - "--source", params.source, - "--weights", params.weights, - "--conf", str(params.conf), - "--iou", str(params.iou), - "--output", "runs/infer" - ] + from sensorflow.execution_ledger import create_execution, mark_running, finalize, append_log + from sensorflow.execution_ops import discover_images, artifact_info + + weights = Path(params.weights) + source = Path(params.source) + out_dir = Path("runs/infer") + + record = create_execution( + "inference", + configuration_snapshot=params.dict(), + input_artifacts=[params.source, params.weights], + command=[ + sys.executable, "infer.py", + "--source", params.source, + "--weights", params.weights, + "--conf", str(params.conf), + "--iou", str(params.iou), + "--output", "runs/infer", + ], + ) + + if not weights.exists() and not str(params.weights).endswith(".pt"): + finalize( + record["execution_id"], + "FAILED", + errors=[f"Model checkpoint not found: {params.weights}"], + process_invoked=False, + outputs_valid=False, + ) + raise HTTPException( + status_code=400, + detail={"message": f"Model checkpoint not found: {params.weights}", "execution_id": record["execution_id"]}, + ) + if not weights.exists(): + # Named ultralytics weight may auto-download; still warn if missing before run + pass + + discovery = discover_images(params.source) + if discovery["images_discovered"] == 0: + finalize( + record["execution_id"], + "FAILED", + records_discovered=0, + records_processed=0, + records_succeeded=0, + records_failed=0, + errors=[f"No images found at source: {params.source}"], + metrics={"images_discovered": 0}, + process_invoked=False, + outputs_valid=False, + ) + return { + "status": "FAILED", + "execution_id": record["execution_id"], + "images": [], + "records_discovered": 0, + "records_succeeded": 0, + "records_failed": 0, + "message": f"No images found at source: {params.source}", + "model": params.weights, + "checkpoint": artifact_info(weights), + } + + if not weights.exists(): + finalize( + record["execution_id"], + "FAILED", + errors=[f"Missing model checkpoint: {params.weights}"], + records_discovered=discovery["images_discovered"], + process_invoked=False, + outputs_valid=False, + ) + raise HTTPException( + status_code=400, + detail={ + "message": f"Missing model checkpoint: {params.weights}", + "execution_id": record["execution_id"], + }, + ) + cmd = record["command"] + mark_running(record["execution_id"]) try: - res = subprocess.run(cmd, capture_output=True, text=True, check=True) - except subprocess.CalledProcessError as e: - raise HTTPException(status_code=500, detail=f"Inference script failed: {e.stdout}\n{e.stderr}") + res = subprocess.run(cmd, capture_output=True, text=True) + except Exception as e: + finalize( + record["execution_id"], + "FAILED", + errors=[str(e)], + process_invoked=True, + outputs_valid=False, + ) + raise HTTPException(status_code=500, detail=f"Inference failed to start: {e}") + + if res.stdout: + append_log(record["execution_id"], res.stdout[-4000:]) + if res.stderr: + append_log(record["execution_id"], res.stderr[-2000:]) - # Gather list of output annotated images - out_dir = Path("runs/infer") images = [] if out_dir.exists(): images = [f.name for f in out_dir.glob("annotated_*") if f.is_file()] + preds_path = out_dir / "predictions.json" + pred_count = 0 + if preds_path.exists(): + try: + pred_count = len(json.loads(preds_path.read_text())) + except Exception: + pred_count = 0 + + succeeded = len(images) + failed = max(0, discovery["images_discovered"] - succeeded) + if res.returncode != 0: + status = "FAILED" + elif succeeded == 0: + status = "FAILED" + elif failed > 0: + status = "PARTIAL_SUCCESS" + else: + status = "SUCCEEDED" + + final = finalize( + record["execution_id"], + status, + exit_code=res.returncode, + records_discovered=discovery["images_discovered"], + records_processed=discovery["images_discovered"], + records_succeeded=succeeded, + records_failed=failed, + metrics={ + "model": params.weights, + "checkpoint": artifact_info(weights), + "inference_calls": discovery["images_discovered"], + "predictions_generated": pred_count, + "annotated_images": succeeded, + "conf": params.conf, + "iou": params.iou, + "output_dir": str(out_dir), + }, + output_artifacts=[ + {"path": str(out_dir), "kind": "infer_dir"}, + artifact_info(preds_path), + ], + errors=[] if res.returncode == 0 else [res.stderr or res.stdout or "infer non-zero exit"], + process_invoked=True, + outputs_valid=succeeded > 0 and res.returncode == 0, + ) - return {"status": "ok", "images": sorted(images)} + return { + "status": final["status"], + "execution_id": final["execution_id"], + "verified": final.get("verified"), + "duration_ms": final.get("duration_ms"), + "images": sorted(images), + "model": params.weights, + "checkpoint": artifact_info(weights), + "records_discovered": discovery["images_discovered"], + "records_processed": discovery["images_discovered"], + "records_succeeded": succeeded, + "records_failed": failed, + "inference_calls": discovery["images_discovered"], + "predictions_generated": pred_count, + "output_dir": str(out_dir), + "exit_code": res.returncode, + "process_id": None, + "command": cmd, + } @app.get("/api/images/{filename}") def serve_image(filename: str): @@ -283,76 +714,146 @@ def serve_image(filename: str): @app.get("/api/grade") def grade_predictions(): - # Runs the autograder.py script + from sensorflow.execution_ledger import create_execution, mark_running, finalize + from sensorflow.execution_ops import artifact_info + predictions_file = Path("runs/infer/predictions.json") + record = create_execution( + "auto_grader", + configuration_snapshot={"predictions": str(predictions_file)}, + input_artifacts=[str(predictions_file)], + command=[sys.executable, "autograder.py", "--predictions", str(predictions_file)], + ) + if not predictions_file.exists(): - # Let's run a fallback report if no predictions exist + final = finalize( + record["execution_id"], + "NOT_EXECUTED", + errors=["predictions.json missing — run Auto-Labeler Inference first"], + process_invoked=False, + outputs_valid=False, + warnings=["Grader not executed: no predictions artifact"], + ) return { + "status": "NOT_EXECUTED", + "execution_id": final["execution_id"], "total_predictions": 0, "total_images": 0, - "quality_score": 0.0, + "quality_score": None, + "metrics": {}, "issues": [ { "severity": "WARNING", "type": "Missing predictions", - "description": "Please run the Auto-Labeler Inference step first to generate prediction files.", - "recommendation": "Go back to Step 4 and run Auto-Labeler." + "description": "No predictions.json — grader was NOT executed.", + "recommendation": "Run Auto-Labeler Inference first.", } - ] + ], + "message": "NOT_EXECUTED: missing predictions artifact", } - cmd = [ - sys.executable, "autograder.py", - "--predictions", str(predictions_file) - ] - + mark_running(record["execution_id"]) + cmd = record["command"] try: - subprocess.run(cmd, capture_output=True, text=True, check=True) - except subprocess.CalledProcessError as e: - raise HTTPException(status_code=500, detail=f"Autograder failed: {e.stderr}") + res = subprocess.run(cmd, capture_output=True, text=True) + except Exception as e: + finalize( + record["execution_id"], + "FAILED", + errors=[str(e)], + process_invoked=True, + outputs_valid=False, + ) + raise HTTPException(status_code=500, detail=f"Autograder failed to start: {e}") - # Read generated quality report report_file = Path("runs/infer/quality_report.json") - if report_file.exists(): - with open(report_file, "r") as f: - report_data = json.load(f) - - # Map standard report issues to list - issues = [] - for issue_type, count in report_data.get("issue_summary", {}).items(): - if count > 0: - severity = "WARNING" if issue_type in ["low_confidence", "overlapping_different_class"] else "INFO" - desc = f"Found {count} occurrences of {issue_type.replace('_', ' ')}." - rec = "Retrain model with more samples or tune confidence threshold." - if issue_type == "small_detection": - rec = "Verify that extremely small bounding boxes are correct." - elif issue_type == "class_imbalance": - rec = "Balance class frequency in dataset to prevent bias." - - issues.append({ - "severity": severity, - "type": issue_type.upper().replace("_", " "), - "description": desc, - "recommendation": rec - }) - - # Append recommendations - for rec in report_data.get("recommendations", []): - issues.append({ - "severity": "INFO", - "type": "RECOMMENDATION", - "description": rec, - "recommendation": "Review annotation guidelines." - }) - - return { - "total_predictions": report_data.get("total_predictions", 0), - "total_images": report_data.get("total_images", 0), - "quality_score": report_data.get("quality_score", 0.0), - "issues": issues - } + if res.returncode != 0 or not report_file.exists(): + finalize( + record["execution_id"], + "FAILED", + exit_code=res.returncode, + errors=[res.stderr or res.stdout or "grader produced no report"], + process_invoked=True, + outputs_valid=False, + ) + raise HTTPException( + status_code=500, + detail={ + "message": f"Autograder failed: {res.stderr}", + "execution_id": record["execution_id"], + }, + ) + + with open(report_file, "r") as f: + report_data = json.load(f) + + issues = [] + for issue_type, count in report_data.get("issue_summary", {}).items(): + if count > 0: + severity = "WARNING" if issue_type in ["low_confidence", "overlapping_different_class"] else "INFO" + desc = f"Found {count} occurrences of {issue_type.replace('_', ' ')}." + rec = "Retrain model with more samples or tune confidence threshold." + if issue_type == "small_detection": + rec = "Verify that extremely small bounding boxes are correct." + elif issue_type == "class_imbalance": + rec = "Balance class frequency in dataset to prevent bias." + issues.append({ + "severity": severity, + "type": issue_type.upper().replace("_", " "), + "description": desc, + "recommendation": rec, + }) + + for rec in report_data.get("recommendations", []): + issues.append({ + "severity": "INFO", + "type": "RECOMMENDATION", + "description": rec, + "recommendation": "Review annotation guidelines.", + }) + + metrics = { + "total_predictions": report_data.get("total_predictions", 0), + "total_images": report_data.get("total_images", 0), + "quality_score": report_data.get("quality_score"), + "issue_summary": report_data.get("issue_summary", {}), + "precision": report_data.get("precision"), + "recall": report_data.get("recall"), + "f1": report_data.get("f1"), + "mAP": report_data.get("mAP"), + "tp": report_data.get("tp"), + "fp": report_data.get("fp"), + "fn": report_data.get("fn"), + "per_class": report_data.get("per_class"), + } + # Strip Nones so UI only shows grader-provided metrics + metrics = {k: v for k, v in metrics.items() if v is not None} + + final = finalize( + record["execution_id"], + "SUCCEEDED", + exit_code=res.returncode, + records_discovered=metrics.get("total_images", 0), + records_processed=metrics.get("total_images", 0), + records_succeeded=metrics.get("total_images", 0), + metrics=metrics, + output_artifacts=[artifact_info(report_file), artifact_info(predictions_file)], + process_invoked=True, + outputs_valid=True, + ) - return {"status": "error", "message": "Failed to read grader output"} + return { + "status": final["status"], + "execution_id": final["execution_id"], + "verified": final.get("verified"), + "duration_ms": final.get("duration_ms"), + "total_predictions": metrics.get("total_predictions", 0), + "total_images": metrics.get("total_images", 0), + "quality_score": metrics.get("quality_score"), + "metrics": metrics, + "issues": issues, + "message": "Grader completed from predictions.json", + } @app.post("/api/export") def export_weights(params: ExportParams): @@ -823,23 +1324,62 @@ DATASET_METADATA_STORE = { @app.get("/api/dataset/details") def get_dataset_details(type: str = "local"): - metadata = DATASET_METADATA_STORE.get(type, DATASET_METADATA_STORE["local"]) + metadata = dict(DATASET_METADATA_STORE.get(type, DATASET_METADATA_STORE["local"])) + # Catalog KPIs are reference stats for the selected dataset type — not proof that + # frames are on disk or browsable in Studio. + metadata["browsable"] = False + metadata["catalog_only"] = True + metadata["browse_hint"] = ( + "These percentages are catalog estimates. Use Validate & Browse Images Path " + "or run 3D Ingest, then open Pipeline Outputs to view real frames." + ) return {"status": "ok", "metadata": metadata} @app.post("/api/dataset/preprocess") def preprocess_dataset(params: dict): + """Catalog-only path kept for backward compatibility. + + Prefer POST /api/dataset/load for real discovery evidence. + If source_path is provided, delegates to real load. + """ + if params.get("source_path") or params.get("real_load"): + return dataset_load(params) + dataset_type = params.get("dataset_type", "local") - import time - time.sleep(0.3) - - meta = DATASET_METADATA_STORE.get(dataset_type, DATASET_METADATA_STORE["local"]) - + meta = dict(DATASET_METADATA_STORE.get(dataset_type, DATASET_METADATA_STORE["local"])) + meta["browsable"] = False + meta["catalog_only"] = True + meta["browse_hint"] = ( + "Catalog metadata only — NOT disk load. Use Load & Preprocess or " + "Validate & Browse Images Path for verifiable counts." + ) + from sensorflow.execution_ledger import create_execution, finalize + + record = create_execution( + "dataset_catalog_metadata", + configuration_snapshot={"dataset_type": dataset_type}, + status="QUEUED", + ) + finalize( + record["execution_id"], + "NOT_EXECUTED", + warnings=["Catalog metadata echo only — no discovery/decode ran"], + metrics={"catalog_only": True, "ingestion_pct_is_catalog": True}, + process_invoked=False, + outputs_valid=False, + ) return { - "status": "ok", - "message": f"Dataset {meta['name']} loaded and preprocessed successfully.", + "status": "NOT_EXECUTED", + "execution_id": record["execution_id"], + "message": ( + f"Catalog metadata for {meta['name']} — not the same as loading " + f"browsable frames. Use Load & Preprocess for real evidence." + ), "dataset_type": dataset_type, - "total_frames": meta["loaded_rows"], - "metadata": meta + "total_frames": None, + "browsable": False, + "catalog_only": True, + "metadata": meta, } @app.post("/api/dataset/save-pipeline-tools") @@ -1273,8 +1813,10 @@ def annotate_street(req: AnnotateStreetRequest): # ----------------------------------------------------------------------- class IngestParams(BaseModel): - vendors: List[str] = ["alpamayo", "waymo"] + vendors: List[str] = ["alpamayo", "waymo", "a2d2"] sequence_id: str = "seq_001" + source_path: Optional[str] = None + max_frames: Optional[int] = None class AutoLabelParams(BaseModel): sequence_id: str = "seq_001" @@ -1295,20 +1837,45 @@ class GateParams(BaseModel): @app.post("/api/dataset/ingest") def ingest_dataset(params: IngestParams): from sensorflow.dataset_fusion_engine import DatasetFusionEngine + config = load_config() + source_path = params.source_path if params.source_path is not None else config.source_path try: engine = DatasetFusionEngine() - sequence = engine.ingest(params.vendors, params.sequence_id) + sequence = engine.ingest( + params.vendors, + params.sequence_id, + source_path=source_path, + max_frames=params.max_frames, + ) manifest_path = engine.save_manifest(sequence) + except FileNotFoundError as e: + raise HTTPException(status_code=400, detail=str(e)) except Exception as e: raise HTTPException(status_code=500, detail=f"Ingest failed: {str(e)}") - config = load_config() config.sequence_id = params.sequence_id config.vendors = params.vendors + if params.source_path is not None: + config.source_path = params.source_path with open(CONFIG_PATH, "w") as f: json.dump(config.model_dump(), f, indent=2) - return {"status": "ok", "manifest": str(manifest_path), "sequence_id": params.sequence_id} + frame_count = len(sequence.frames) + demo_stub = bool(sequence.taxonomy_manifest.get("demo_stub")) + return { + "status": "ok", + "manifest": str(manifest_path), + "sequence_id": params.sequence_id, + "frames": frame_count, + "demo_stub": demo_stub, + "vendor": sequence.vendor, + "source_path": source_path, + "message": ( + f"demo stub: {frame_count} frames" + if demo_stub + else f"ingested {frame_count} frames from {source_path}" + ), + } @app.get("/api/dataset/ingest/status") @@ -1323,6 +1890,9 @@ def ingest_status(sequence_id: str = "seq_001"): status["manifest_exists"] = True status["frames"] = len(manifest_data.get("frames", [])) status["vendor"] = manifest_data.get("vendor", "unknown") + tax = manifest_data.get("taxonomy_manifest") or {} + status["demo_stub"] = bool(tax.get("demo_stub", status.get("demo_stub", False))) + status["stub_note"] = tax.get("stub_note") return {"status": "ok", "sequence_id": sequence_id, **status} @@ -1330,13 +1900,43 @@ def ingest_status(sequence_id: str = "seq_001"): def auto_label(params: AutoLabelParams): from sensorflow.perception_automator import PerceptionAutomator from sensorflow.schemas.unified_frame import UnifiedSequence + from sensorflow.execution_ledger import create_execution, mark_running, finalize + from sensorflow.execution_ops import artifact_info manifest = Path("runs/pipeline") / params.sequence_id / "manifest.json" + record = create_execution( + "auto_label_3d", + configuration_snapshot=params.dict(), + input_artifacts=[str(manifest), params.sam_checkpoint], + ) + if not manifest.exists(): - raise HTTPException(status_code=400, detail="Manifest not found. Run ingest first.") + finalize( + record["execution_id"], + "FAILED", + errors=["Manifest not found. Run ingest first."], + process_invoked=False, + outputs_valid=False, + ) + raise HTTPException( + status_code=400, + detail={"message": "Manifest not found. Run ingest first.", "execution_id": record["execution_id"]}, + ) + + ckpt = Path(params.sam_checkpoint) + sam_ran = False + model_name = "sam_vit_b" if not params.no_sam else "none (no_sam)" + if params.no_sam: + warnings = ["no_sam=true — SAM not invoked; proposals may be GT/synthetic fallback"] + elif not ckpt.exists(): + warnings = [f"SAM checkpoint missing: {params.sam_checkpoint} — model will not run"] + else: + warnings = [] + mark_running(record["execution_id"]) try: sequence = UnifiedSequence.load(manifest) + expected_frames = len(sequence.frames) output_dir = manifest.parent / "proposals" automator = PerceptionAutomator( sam_checkpoint=params.sam_checkpoint, @@ -1344,12 +1944,93 @@ def auto_label(params: AutoLabelParams): use_sam=not params.no_sam, ) automator.run_sequence(sequence, output_dir) + sam_ran = bool(getattr(automator, "use_sam", False) and automator._mask_generator is not None) except Exception as e: - raise HTTPException(status_code=500, detail=f"Auto-label failed: {str(e)}") + finalize( + record["execution_id"], + "FAILED", + errors=[str(e)], + process_invoked=False, + outputs_valid=False, + ) + raise HTTPException( + status_code=500, + detail={"message": f"Auto-label failed: {str(e)}", "execution_id": record["execution_id"]}, + ) proposals_dir = manifest.parent / "proposals" - num_frames = len(list(proposals_dir.glob("*.json"))) if proposals_dir.exists() else 0 - return {"status": "ok", "proposals_dir": str(proposals_dir), "frames_processed": num_frames} + proposal_files = list(proposals_dir.glob("*.json")) if proposals_dir.exists() else [] + num_frames = len(proposal_files) + pred_count = 0 + for pf in proposal_files: + try: + data = json.loads(pf.read_text()) + if isinstance(data, list): + pred_count += len(data) + elif isinstance(data, dict): + pred_count += len(data.get("proposals") or data.get("objects") or []) + except Exception: + pass + + demo_stub = bool(sequence.taxonomy_manifest.get("demo_stub")) + if params.no_sam or not sam_ran: + status = "NOT_EXECUTED" if num_frames == 0 else "PARTIAL_SUCCESS" + if params.no_sam: + warnings.append("Model not run (no_sam); do not treat as SAM SUCCESS") + elif not sam_ran: + warnings.append("SAM unavailable — fallback proposals only") + elif num_frames == 0: + status = "FAILED" + elif num_frames < expected_frames: + status = "PARTIAL_SUCCESS" + else: + status = "SUCCEEDED" + + final = finalize( + record["execution_id"], + status, + records_discovered=expected_frames, + records_processed=num_frames, + records_succeeded=num_frames, + records_failed=max(0, expected_frames - num_frames), + metrics={ + "model": model_name, + "checkpoint": artifact_info(ckpt), + "sam_ran": sam_ran, + "no_sam": params.no_sam, + "frames_expected": expected_frames, + "frames_processed": num_frames, + "predictions_generated": pred_count, + "inference_calls": num_frames if sam_ran else 0, + "output_dir": str(proposals_dir), + "demo_stub": demo_stub, + }, + output_artifacts=[{"path": str(proposals_dir), "kind": "proposals_dir", "files": num_frames}], + warnings=warnings, + process_invoked=sam_ran, + outputs_valid=num_frames > 0, + ) + + return { + "status": final["status"], + "execution_id": final["execution_id"], + "verified": final.get("verified"), + "duration_ms": final.get("duration_ms"), + "proposals_dir": str(proposals_dir), + "frames_processed": num_frames, + "frames_expected": expected_frames, + "predictions_generated": pred_count, + "inference_calls": num_frames if sam_ran else 0, + "model": model_name, + "checkpoint": artifact_info(ckpt), + "sam_ran": sam_ran, + "demo_stub": demo_stub, + "message": ( + f"{'NOT_EXECUTED / fallback' if not sam_ran else 'SAM'}: " + f"processed {num_frames}/{expected_frames} frames, {pred_count} predictions → {proposals_dir}" + ), + "events": final.get("events"), + } @app.post("/api/perception/track") @@ -1440,18 +2121,109 @@ def pipeline_status(sequence_id: str = "seq_001"): seq_state = state.get(sequence_id, {}) base = Path("runs/pipeline") / sequence_id + manifest = base / "manifest.json" + proposals_dir = base / "proposals" + tracks = base / "tracks.json" + benchmark = base / "benchmark" / "metric_card.json" + quality_report = base / "benchmark" / "quality_report.json" + + ingest_complete = manifest.exists() + perception_complete = proposals_dir.exists() and any(proposals_dir.glob("*.json")) + tracking_complete = tracks.exists() + benchmark_complete = benchmark.exists() + + frames_ingested = None + frames_processed = None + demo_stub = None + if ingest_complete: + try: + with open(manifest) as f: + manifest_data = json.load(f) + frames_ingested = len(manifest_data.get("frames", [])) + demo_stub = bool((manifest_data.get("taxonomy_manifest") or {}).get("demo_stub")) + except Exception: + frames_ingested = seq_state.get("frames") + demo_stub = seq_state.get("demo_stub") + if perception_complete: + frames_processed = len(list(proposals_dir.glob("*.json"))) + + # Completion stages: True when done, None when not run (UI must not show FAIL). + # Launch gate: True/False only after evaluation; None beforehand. + launch_gate_passed = seq_state.get("launch_gate_passed") + if "launch_gate_passed" not in seq_state: + launch_gate_passed = None + + quality_passed = None + if quality_report.exists(): + try: + with open(quality_report) as f: + quality_passed = bool(json.load(f).get("passed")) + except Exception: + quality_passed = benchmark_complete + return { "status": "ok", "sequence_id": sequence_id, - "ingest_complete": (base / "manifest.json").exists(), - "perception_complete": (base / "proposals").exists() and any((base / "proposals").glob("*.json")), - "tracking_complete": (base / "tracks.json").exists(), - "benchmark_complete": (base / "benchmark" / "metric_card.json").exists(), - "launch_gate_passed": seq_state.get("launch_gate_passed", False), + "ingest_complete": True if ingest_complete else None, + "perception_complete": True if perception_complete else None, + "tracking_complete": True if tracking_complete else None, + "benchmark_complete": True if benchmark_complete else None, + "quality_gate_passed": quality_passed, + "launch_gate_passed": launch_gate_passed, + "frames_ingested": frames_ingested, + "frames_processed": frames_processed, + "demo_stub": demo_stub, "state": seq_state, } +@app.get("/api/pipeline/sequences") +def pipeline_sequences(): + from sensorflow import pipeline_artifacts as artifacts + return {"status": "ok", "sequences": artifacts.list_sequences()} + + +@app.get("/api/pipeline/artifacts") +def pipeline_artifacts(sequence_id: str = "seq_001"): + from sensorflow import pipeline_artifacts as artifacts + return {"status": "ok", **artifacts.artifacts_summary(sequence_id)} + + +@app.get("/api/pipeline/frames") +def pipeline_frames(sequence_id: str = "seq_001"): + from sensorflow import pipeline_artifacts as artifacts + return artifacts.list_frames(sequence_id) + + +@app.get("/api/pipeline/frame") +def pipeline_frame(sequence_id: str, frame_id: str): + from sensorflow import pipeline_artifacts as artifacts + try: + return artifacts.get_frame(sequence_id, frame_id) + except FileNotFoundError as e: + raise HTTPException(status_code=404, detail=str(e)) + + +@app.get("/api/pipeline/file") +def pipeline_file(path: str): + """Serve a local run/data artifact with path-traversal protection.""" + from sensorflow import pipeline_artifacts as artifacts + try: + file_path, media_type = artifacts.resolve_file(path) + except PermissionError as e: + raise HTTPException(status_code=403, detail=str(e)) + except FileNotFoundError as e: + raise HTTPException(status_code=404, detail=str(e)) + return FileResponse(file_path, media_type=media_type) + + +@app.get("/api/dataset/browse") +def dataset_browse(source_path: str = "data", limit: int = 48): + """Validate Images Path by listing browsable local frames.""" + from sensorflow import pipeline_artifacts as artifacts + return artifacts.scan_source_images(source_path, limit=limit) + + @app.get("/api/mitl/queue") def get_mitl_queue(sequence_id: str = "seq_001"): from sensorflow.mitl_copilot import MitlCopilot @@ -1534,6 +2306,14 @@ app.include_router(studio2_router) from sensorflow.rotr.api import router as rotr_router app.include_router(rotr_router) +# In-app help chatbot (FAQ / page-guide matcher; optional Ollama enrichment). +from sensorflow.help.api import router as help_router +app.include_router(help_router) + +# Product version + About catalog (GET /api/about, GET /api/version). +from sensorflow.about.api import router as about_router +app.include_router(about_router) + # Mount static folder static_dir = Path(__file__).parent / "static" if static_dir.exists(): diff --git a/docs/execution-integrity-audit.md b/docs/execution-integrity-audit.md new file mode 100644 index 0000000000000000000000000000000000000000..c03f5b4fa4a55c47ca13e443f4fa49248da772db --- /dev/null +++ b/docs/execution-integrity-audit.md @@ -0,0 +1,99 @@ +# Sensorflow Studio — Execution Integrity & Observability Audit + +**Date:** 2026-08-14 +**Scope:** Legacy Studio (`static/`, `app_backend.py`, `sensorflow/`, `train.py`, `infer.py`, `autograder.py`) +**Mirror:** `hf-space/` copies are effectively identical for these surfaces — findings apply to both. +**Prime directive:** A UI state is NOT evidence. PASS / Done / 100% / Found / `success:true` / toasts / YAML alone / file existence alone are not proof of execution. + +--- + +## Stage integrity table + +| COMPONENT | CURRENT BEHAVIOR | ACTUAL EXECUTION? | EVIDENCE | RISK | REQUIRED FIX | +|-----------|------------------|-------------------|----------|------|--------------| +| 1 Dataset Configuration / Load & Preprocess | Save config writes JSON; Pre-Check = script file existence; “Load Catalog Metadata” returns hardcoded KPIs (`ingestion_pct: "100.0%"`) after `sleep(0.3)`; Validate & Browse is real disk scan. Toast “saved & verified” without path checks. | **Partial** — browse real; catalog preprocess is not load | `app_backend.py` precheck/details/preprocess; `static/app.js` save/browse handlers | **High** | Real Load & Preprocess with discovery counts + ledger; never show 100% Loaded from catalog | +| 2 3D Ingest & Fusion | Calls fusion engine; vendor adapters default `demo_stub`; local path can be real. UI PASS when `manifest.json` exists. | **Yes local**; **stub vendors** | ingest API + adapters; gate badges | **Medium** | PASS/DONE only with provenance; surface `demo_stub` + execution_id | +| 3 3D Perception / Auto-label (SAM) | Frontend hardcodes `no_sam: true`. Automator falls back to GT/synthetic masks; still `status: ok` → PASS. | **No real SAM** on Studio UI path | `static/app.js` perception handler; `perception_automator.py` | **Critical** | Record model/checkpoint/counts; NOT_EXECUTED if model not run; refuse fake SUCCESS | +| 4 Temporal Tracking | Real Kalman+Hungarian over proposals. | **Yes** (inputs may be fake) | track API | **Medium** | Propagate contamination; ledger entry | +| 5 Quality Gate | Real metric math vs sequence GT; contaminated proposals inflate pass. | **Yes math**; honesty depends on inputs | quality gate API | **High** | Fail closed on stub/GT-fallback; evidence card | +| 6 Launch Gate | Threshold check on metric card. | **Yes** (inherited) | launch gate evaluator | **Medium** | Same provenance gate | +| 7 Model Setup (2D) | UI model card only; no weight verification. | **No** | model cards in `app.js` | **Low–Medium** | Verify weights exist before train/infer | +| 8 Training Execution | Spawns real `train.py`. Status fabricates losses; forces progress=1.0; ignores exit_code. UI “Process finished” with no verdict. | **Partial** | train start/status; `train.py` | **Critical** | PID, command, exit_code, checkpoint hash, ledger | +| 9 Auto-Labeler Inference | Runs `infer.py` with check=True. Empty source exits 0 → ok + empty images. UI always success toast. | **Partial** | infer API; `infer.py` | **High** | Counts + checkpoint + exit_code; NOT_EXECUTED/FAILED if 0 images / missing weights | +| 10 Auto-Grader Quality | Real autograder when predictions exist; empty issues → “Perfect score!”. | **Yes** (heuristic) | grade API; `autograder.py` | **Medium** | Metrics only from grader output; ledger | +| 11 MITL Review | Hardcoded Unsplash samples; claims HF ingest; Mock Triage Auditor. | **No** | nvidia/load; mitl/evaluate | **Critical** | Label fixtures; never SUCCEEDED for mock ingest | +| 12 Model Benchmarking | `fallback_mock` hardcoded metrics with `status: ok`. | **No** (unless live card) | `/api/benchmark/compare` | **Critical** | Refuse OK without live metrics; stub = NOT_EXECUTED | +| 13 Export & Deploy | May run Ultralytics export; returns path without exists(); UI “✓ complete” without `res.ok`. | **Partial** | export API | **High** | Verify artifact; ledger | +| 14 MCP Settings | Real R/W when file exists; invented servers + “changes simulated”. | **Partial** | mcp APIs | **Medium** | Honest status; evidence stub | +| 15 SSAM Safety | Hardcoded CA intersections presented as statewide FHWA dataset. | **No real lake** | ssam endpoints + catalog | **High** | Label synthetic demo; stub evidence | + +--- + +## High-risk fake-success occurrences + +### Backend (`app_backend.py`) + +| Location | Pattern | +|----------|---------| +| `/api/precheck` | Success = `train.py` / `infer.py` / `autograder.py` **file existence only** | +| `/api/train/status` | Fabricates decreasing losses; forces `progress=1.0`; ignores process `returncode` | +| `/api/infer/run` | Empty run still `status: ok` when script exits 0 with 0 images | +| `/api/export` | Returns export path without verifying file exists | +| `/api/nvidia/load` | Hardcoded samples; claims HF ingest success | +| MITL evaluate | Mock Triage Auditor returned as success-shaped response | +| `/api/benchmark/compare` | `fallback_mock` hardcoded metrics, `status: ok` | +| `DATASET_METADATA_STORE` | Hardcoded `ingestion_pct: "100.0%"` catalog KPIs | +| `/api/dataset/preprocess` | `time.sleep(0.3)` + catalog echo — no disk load | +| MCP get/toggle | Invented config / “changes simulated” with ok/warning | +| SSAM store | Hardcoded intersections as statewide dataset | +| `/api/pipeline/status` | File existence ⇒ `*_complete: true` | + +### Frontend (`static/app.js`) + +| Location | Pattern | +|----------|---------| +| Gate badges | Completion stages → **PASS** from file-complete; never FAIL | +| Config save | “verified successfully” without `res.ok` / path validation | +| Training poll | “Process finished” with no exit_code / success check | +| Infer | Always success toast (even empty / `!res.ok`) | +| Grader | Empty issues → “Perfect score!” | +| Export | “✓ Export complete” without `res.ok` | +| Perception | **`no_sam: true` hardcoded** | + +### Scripts / sensorflow + +| Location | Pattern | +|----------|---------| +| `train.py` dgx-spark | Prints remote offload; uses local device | +| `infer.py` | No images → print and **exit 0** | +| `perception_automator.py` | Silent SAM off; synthetic masks/LiDAR; GT class copy | +| Vendor adapters | Built-in ~3-frame `demo_stub` lakes | + +--- + +## Cross-cutting patterns + +1. **`status: "ok"` / success toast without evidence** +2. **File existence = done/PASS** +3. **Hardcoded metrics / “100%” catalog KPIs** +4. **Simulated delays / stubs claiming complete** +5. **Subprocess integrity gaps** (exit ignored, empty exit 0) +6. **GT contamination loop** (`no_sam` → synthetic/GT proposals → quality/launch PASS) +7. **Frontend rarely checks `res.ok`** + +--- + +## What is relatively honest today + +- Validate & Browse (`/api/dataset/browse`) — real filesystem scan +- Local ingest (non-stub) — real frames → manifest +- Tracking / quality math / launch thresholds / autograder heuristics / YOLO train+infer — real when inputs and deps exist +- Catalog preprocess messaging partially hardened (`catalog_only`, `browse_hint`) but KPIs still read as Loaded + +--- + +## Remediation shipped in `feat/execution-integrity` + +**Implemented:** Execution ledger + APIs; real Dataset Load & Preprocess; Auto-Label / Infer / Train / Grader evidence wiring; YAML semantic validation; script verification; backend health; Strict Mode; Evidence Cards + Global Execution Console; failure + deterministic tests; remaining stages show honest `NOT_EXECUTED` / unverified stubs. + +**Deferred (honest stubs only):** Full MITL/HF ingest, real SSAM lake, real MCP process probes, SAM GPU path when checkpoint absent, full 15-stage process verification beyond Dataset/Auto-Label/Train/Grader. diff --git a/hf-space/.dockerignore b/hf-space/.dockerignore deleted file mode 100644 index 512dddf307594855c676bee507c674570d52f738..0000000000000000000000000000000000000000 --- a/hf-space/.dockerignore +++ /dev/null @@ -1,12 +0,0 @@ -node_modules/ -.venv/ -.git/ -runs/ -dist/ -models/ -*.pt -__pycache__/ -**/__pycache__/ -.pytest_cache/ -.claude/ -.DS_Store diff --git a/hf-space/.gitattributes b/hf-space/.gitattributes deleted file mode 100644 index a6344aac8c09253b3b630fb776ae94478aa0275b..0000000000000000000000000000000000000000 --- a/hf-space/.gitattributes +++ /dev/null @@ -1,35 +0,0 @@ -*.7z filter=lfs diff=lfs merge=lfs -text -*.arrow filter=lfs diff=lfs merge=lfs -text -*.bin filter=lfs diff=lfs merge=lfs -text -*.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.ftz filter=lfs diff=lfs merge=lfs -text -*.gz filter=lfs diff=lfs merge=lfs -text -*.h5 filter=lfs diff=lfs merge=lfs -text -*.joblib filter=lfs diff=lfs merge=lfs -text -*.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text -*.model filter=lfs diff=lfs merge=lfs -text -*.msgpack filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.npz filter=lfs diff=lfs merge=lfs -text -*.onnx filter=lfs diff=lfs merge=lfs -text -*.ot filter=lfs diff=lfs merge=lfs -text -*.parquet filter=lfs diff=lfs merge=lfs -text -*.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text -*.pth filter=lfs diff=lfs merge=lfs -text -*.rar filter=lfs diff=lfs merge=lfs -text -*.safetensors filter=lfs diff=lfs merge=lfs -text -saved_model/**/* filter=lfs diff=lfs merge=lfs -text -*.tar.* filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text -*.tflite filter=lfs diff=lfs merge=lfs -text -*.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text -*.xz filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.zst filter=lfs diff=lfs merge=lfs -text -*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/hf-space/.gitignore b/hf-space/.gitignore deleted file mode 100644 index de0b34bc704869fbd29c1678be2ff7d0003538b0..0000000000000000000000000000000000000000 --- a/hf-space/.gitignore +++ /dev/null @@ -1,72 +0,0 @@ -# Virtual environments -.venv/ -venv/ -env/ -ENV/ - -# Python -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# Testing -.pytest_cache/ -.coverage -htmlcov/ - -# Node -node_modules/ -package-lock.json - -# IDE -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# OS -.DS_Store -Thumbs.db - -# Large model files (download on deployment) -*.pt - -# Logs -*.log -logs/ - -# Local development -.env -.env.local -*.local.json - -# Temporary -/tmp/ -temp/ - -# Generated runtime/test artifacts -runs/labeleval/ -runs/megaeval/ -runs/pipeline/frame_test/ -runs/pipeline/qg_test/ -runs/pipeline/roundtrip/ -runs/pipeline/test_seq/ -runs/pipeline/test_waymo/ diff --git a/hf-space/ANTIGRAVITY_PROMPT.md b/hf-space/ANTIGRAVITY_PROMPT.md deleted file mode 100644 index 21bc3e3ee10d238b61acf79d0bb962fc6af2e52e..0000000000000000000000000000000000000000 --- a/hf-space/ANTIGRAVITY_PROMPT.md +++ /dev/null @@ -1,192 +0,0 @@ -# Accident Analysis Feature Implementation Prompt for Ollama - -**Implementation Model**: `qwen3-coder:30b` (ollama-spark-coding) - -**MCP Host**: `http://dgx-spark.tail16d8d9.ts.net:11434` - ---- - -## Quick Summary - -Implement a production-grade accident/crash analysis feature using two available toolkits: -1. **CrashDataAnalyzer** (`crash_analyzer.py`) — SWITRS data analysis -2. **DataExtractionPipeline** (`data_extractor.py`) — ETL for any source - -Build 6 layers: Ingestion → Validation → Cleaning → Analysis → Insights → API - ---- - -## Direct Prompt for qwen3-coder:30b - -Copy and submit to `qwen3-coder:30b` via Ollama MCP: - -``` -=== ACCIDENT ANALYSIS FEATURE - IMPLEMENTATION === - -Build a production accident/crash analysis platform in Python 3.9+. - -AVAILABLE TOOLKITS (in repository): -- crash_analyzer.py: CrashDataAnalyzer for SWITRS data analysis -- data_extractor.py: DataExtractionPipeline, DataValidator, DataCleaner -- Examples: example_crash_analysis.py, example_data_extraction.py - -BUILD 6 LAYERS (in order): - -LAYER 1: DATA INGESTION (accident_importer.py) -- Class: AccidentDataImporter -- Load from: CSV, APIs (paginated), SQL databases, JSON/Parquet -- Auto-detect format, handle errors, standardize schema -- Base: DataExtractionPipeline - -LAYER 2: DATA VALIDATION (accident_validator.py) -- Class: AccidentDataValidator -- Validate: required fields, types, coordinate ranges, date ranges -- Detect: outliers, missing values, anomalies -- Return: quality report with score -- Base: DataValidator - -LAYER 3: DATA CLEANING (accident_cleaner.py) -- Class: AccidentDataCleaner -- Standardize: column names (snake_case), severity, vehicle types -- Handle: missing values, duplicates, datetime, coordinates -- Base: DataCleaner - -LAYER 4: ANALYSIS ENGINE (accident_analysis.py) -- Classes: MetadataAnalyzer, GeospatialAnalyzer, DemographicsAnalyzer -- Orchestrator: AccidentAnalysisEngine -- Analyze: temporal patterns, hotspots, demographics -- Cache results in memory -- Base: CrashDataAnalyzer - -LAYER 5: INSIGHTS & REPORTING (accident_insights.py, accident_report.py) -- AccidentInsights: deadliest times/places/groups, risk scores -- AccidentReport: HTML/PDF/JSON export, visualizations - -LAYER 6: API & INTEGRATION (main.py, routes/, models/) -- FastAPI endpoints: - POST /api/analyze (upload CSV) - GET /api/hotspots (geographic) - GET /api/insights (findings) - GET /api/report (download) - WebSocket /ws/stream (real-time) - -PRODUCTION REQUIREMENTS: -✓ Complete working code (no TODOs) -✓ Type hints on all functions -✓ Google-style docstrings -✓ Error handling (try/except with logging) -✓ 80%+ test coverage (pytest) -✓ Performance: <5s for 10k records -✓ Security: input validation, SQL injection protection -✓ Logging: DEBUG/INFO/ERROR/CRITICAL levels -✓ Config: environment variables - -OUTPUT EACH LAYER: -1. Complete working Python code -2. Unit tests (pytest) -3. Integration tests -4. Full docstrings & type hints -5. Error handling with examples -6. Usage examples - -Start with LAYER 1. Use existing toolkits. Follow PEP 8. -``` - ---- - -## MCP Server Configuration - -```json -{ - "mcpServers": { - "ollama-spark-coding": { - "command": "npx", - "args": ["-y", "@iflow-mcp/ollama-mcp"], - "env": { - "OLLAMA_HOST": "http://dgx-spark.tail16d8d9.ts.net:11434", - "OLLAMA_MODEL": "qwen3-coder:30b" - } - } - } -} -``` - ---- - -## Model Selection - -**Primary**: `qwen3-coder:30b` (code generation specialist) -- Best for Python implementation across 6 layers -- Understands complex architecture and APIs -- Large context window for full specs - -**Optional**: `reasoning-core` (planning/validation) -- Good for architecture review -- Validates implementation decisions - -**Optional**: `gemma4:26b` (documentation) -- Generate docs, examples, deployment guides - ---- - -## 6-Layer Architecture - -``` -Data Source (CSV/API/DB) - ↓ -LAYER 1: Data Ingestion - ↓ -LAYER 2: Data Validation (Quality Check) - ↓ -LAYER 3: Data Cleaning (Normalization) - ↓ -LAYER 4: Analysis Engine (Metadata/Geo/Demographics) - ↓ -LAYER 5: Insights & Reports (HTML/PDF/JSON) - ↓ -LAYER 6: REST API (FastAPI endpoints) -``` - ---- - -## Key Requirements - -- **Language**: Python 3.9+ -- **Framework**: FastAPI (API layer) -- **Testing**: pytest (80%+ coverage) -- **Data**: SWITRS format (accidents.csv, vehicles.csv, parties.csv) -- **Performance**: <5 seconds for 10k records -- **Code Quality**: Type hints, docstrings, error handling - ---- - -## Timeline - -- Layer 1 (Ingestion): 1-2 days -- Layer 2 (Validation): 1 day -- Layer 3 (Cleaning): 1 day -- Layer 4 (Analysis): 2-3 days -- Layer 5 (Insights): 1-2 days -- Layer 6 (API): 2 days -- Tests & Deployment: 2 days - -**Total**: 11-17 days - ---- - -## Success Criteria - -✅ Extract from 3+ sources (CSV, API, database) -✅ Validate with >95% accuracy -✅ Analyze temporal, geospatial, demographic patterns -✅ Identify hotspots and high-risk groups -✅ Generate HTML/PDF/JSON reports -✅ Provide REST API -✅ Support real-time analysis -✅ Handle 1GB+ datasets -✅ 80%+ test coverage -✅ Production-ready code - ---- - -**Ready for Ollama Implementation**: qwen3-coder:30b diff --git a/hf-space/ARCHITECTURE.md b/hf-space/ARCHITECTURE.md deleted file mode 100644 index bca15fdbdc61e648c64c896b58be3032efeb47e7..0000000000000000000000000000000000000000 --- a/hf-space/ARCHITECTURE.md +++ /dev/null @@ -1,78 +0,0 @@ -# Sensorflow Studio — Aggregate-First Evaluation Architecture (megaeval) - -The `sensorflow/megaeval/` layer replaces the "evaluation-record browser" model -(annotation as the primary unit) with an **aggregate-first, risk-first, -cohort-first, drill-down-on-demand** design. - -## Mental model → code - -| Concept | Meaning | Module / storage | -| --- | --- | --- | -| Dataset | immutable source population | `population.py` → `runs/megaeval/populations/{pop}/part-*.npz` | -| Evaluation Run | model+config being evaluated (async job, first-class) | `runs.py` → `runs/megaeval/runs/{run}/run.json` | -| Population | all objects evaluated in a run | headline row of the metric cube | -| Cohort | slice by class/weather/lighting/road/scenario/sensor/distance/speed/occlusion | cube cells + `POST /api/evaluations/query` | -| Container | scene/frame/segment holding related objects | `containers.npz` aggregate table | -| Annotation | individual object — forensic view only | per-object `objects-part-*.npz`, loaded one partition at a time | - -## Data flow - -``` -generate population (vectorized numpy, partitioned npz) - │ -create run (queued) ──► worker pool (4 threads) processes partitions: - │ model simulation → PARTIAL sufficient statistics - │ (n, tp, fp, fn, sum_iou, conf sums, histograms …) - │ grouped by the 9 cube dimensions, per partition - ▼ -reduce (concat + groupby-sum) ──► materialize METRIC CUBE (cube.npz) - │ + error index, container table, sketches, - │ container embeddings - ▼ -publish ──► queries served by QueryRouter: cache → cube → (rare) record scan -``` - -Aggregation **never** scans raw records at query time; queries like -"pedestrian recall at night in rain for model v42" are cube lookups. - -## Single-node stand-ins for distributed infrastructure - -This repo runs on one laptop; the seams are shaped so each piece maps to the -production-scale equivalent: - -| Here (single node) | Production analogue | -| --- | --- | -| npz column partitions (`part-0000.npz…`, hash-partitioned by container) | Parquet files in an Iceberg/Delta table, partitioned by scene | -| `ThreadPoolExecutor` workers over partitions | Spark executors / Flink task slots | -| partial stats per partition + `reduce_partials` | map-side combine + shuffle reduce | -| `QueryRouter` (cache → cube → scan) with latency+source reporting | OLAP engine (Druid/Pinot/ClickHouse) over the lakehouse | -| `QueryCache` keyed by hash(dataset+model+filters+group_by+metrics) | result cache / materialized views | -| in-process SSE progress events | job-status topic (Kafka) + progress service | -| numpy cosine similarity over container embeddings | vector DB (hybrid retrieval) | - -## Honesty rules encoded in the API - -- Cube counts and derived ratios are **exact** (`meta.exact = true`). -- Percentiles/distributions come from fixed-bin quantile histograms; - cardinalities from HyperLogLog — always labeled approximate. -- Precision/recall from human review are **sampling estimates with 95% CIs** - (stratified risk-weighted design: Wilson per stratum, weighted normal-approx - combination), reported with n reviewed / frame size / method. -- Every query response reports `source` (cache/cube/scan), `cache_hit`, - and `latency_ms` so aggregate-first behavior is observable in the UI. -- Evaluation runs are reproducible: lineage (dataset, model+checkpoint, label - version, evaluator code version, metric version, thresholds, sampling config, - seed, hardware) is stored on the run and derives the deterministic seed. - -## Performance SLOs (measured locally, 320k-object population, 67k cube cells) - -| Query | SLO | Measured | -| --- | --- | --- | -| aggregate dashboard (group by class) | < 2 s | ~75 ms | -| filtered cohort query | < 3 s | ~350 ms first / sub-ms cached | -| cached query | < 100 ms | ~0.1 ms | -| object investigation (one container, forensic) | < 1 s | ~50–200 ms | - -The legacy `sensorflow/evaluation/` layer (label-evaluation pipeline, triage, -HITL, training flywheel) remains unchanged; megaeval integrates with its -regression tracker and alerting rather than duplicating them. diff --git a/hf-space/CRASH_DATA_GUIDE.md b/hf-space/CRASH_DATA_GUIDE.md deleted file mode 100644 index ff5bb02d9d833dc372cde876bd3fd244ec798c1a..0000000000000000000000000000000000000000 --- a/hf-space/CRASH_DATA_GUIDE.md +++ /dev/null @@ -1,378 +0,0 @@ -# Crash Data Analysis Guide - -Complete guide to analyzing crash data with metadata, geospatial, and vehicle/traveler dimensions. - -## Quick Start - -```python -from crash_analyzer import load_switrs_data - -# Load SWITRS CSV data -analyzer = load_switrs_data('accidents.csv', 'vehicles.csv', 'parties.csv') - -# Generate full report -report = analyzer.generate_summary_report() -``` - -## Data Source: SWITRS - -**Reference**: https://tims.berkeley.edu/help/crashdata/switrs-geocoding.php - -SWITRS (Statewide Integrated Traffic Records System) provides comprehensive California highway crash data with three main record types: - -### 1. Accidents Table (Core Records) -Primary crash incident information. - -| Field | Type | Description | Example | -|---|---|---|---| -| `case_id` | int | Unique crash identifier | 123456 | -| `accident_date` | date | Date of crash | 2024-01-15 | -| `accident_time` | time | Time of crash (24-hr) | 14:30 | -| `latitude` | float | Geocoded latitude | 37.7749 | -| `longitude` | float | Geocoded longitude | -122.4194 | -| `county` | string | County name | "San Francisco" | -| `district` | string | CHP district | "SF" | -| `severity` | string | Crash severity | F/I/P (Fatal/Injury/PropertyDamage) | -| `collision_type` | string | Type of collision | "Rear End", "Broadside", etc. | -| `weather_1` | string | Primary weather | "Clear", "Rain", "Fog" | -| `weather_2` | string | Secondary weather | Optional | -| `surface_condition` | string | Road surface | "Dry", "Wet", "Snow" | -| `traffic_control_device` | string | Traffic control status | "Functioning", "Not Functioning" | -| `fatality_count` | int | Number of fatalities | 0-5 | -| `injury_count` | int | Number of injuries | 0-10 | - -### 2. Vehicles Table (Vehicle Details) -Vehicle information for each crash. - -| Field | Type | Description | -|---|---|---| -| `case_id` | int | Link to accidents table | -| `vehicle_id` | int | Unique vehicle in crash | -| `vehicle_type` | string | Car, Truck, Motorcycle, SUV, Van, etc. | -| `vehicle_year` | int | Vehicle model year | -| `vehicle_make` | string | Manufacturer (Toyota, Ford, etc.) | -| `vehicle_model` | string | Model name | -| `vehicle_color` | string | Color | -| `damage_extent` | string | Severe, Moderate, Minor | -| `direction_of_travel` | string | N, S, E, W, etc. | - -### 3. Parties Table (Occupants & Pedestrians) -Information about crash participants (drivers, passengers, pedestrians). - -| Field | Type | Description | -|---|---|---| -| `case_id` | int | Link to accidents table | -| `party_number` | int | Sequence in crash | -| `party_type` | string | Driver, Passenger, Pedestrian, Cyclist | -| `age` | int | Age in years | -| `gender` | string | M/F | -| `sobriety` | int | 1=Sober, 2=Drinking, 3=Intoxicated, 4=Unknown | -| `injury_level` | int | 0=None, 1=Possible, 2=Probable, 3=Severe, 4=Fatal | -| `safety_equipment_used` | string | Seatbelt, Helmet, etc. | -| `ejection` | string | Yes/No/Unknown | -| `primary_collision_factor` | string | Cause (Speeding, DUI, etc.) | - ---- - -## Analysis Dimensions - -### 1. METADATA ANALYSIS - -**Time-Based Patterns** -```python -# Identify dangerous times -temporal = analyzer.analyze_temporal_patterns() -# Returns: crashes by hour, day, month, year + deadliest times -``` - -**What to Look For**: -- Rush hour peaks (7-9am, 4-6pm) -- Weekend spikes -- Holiday effect (New Year, July 4th) -- Seasonal trends (winter weather, spring break) - -**Severity Analysis** -```python -severity = analyzer.analyze_severity() -# Returns: distribution of Fatal/Injury/Property Damage -``` - -**What to Report**: -- Fatality rate (total deaths / total crashes) -- Injury-to-crash ratio -- Crash trend over time (increasing/decreasing) - -**Collision Types** -```python -collisions = analyzer.analyze_collision_types() -# Returns: most common collision types and deadliest types -``` - -**Key Metrics**: -- Most common collision type -- Deadliest collision type (by fatality rate) -- Preventable vs. unpreventable patterns - ---- - -### 2. GEOSPATIAL ANALYSIS - -**Geographic Summaries** -```python -geo = analyzer.get_geographic_summary() -# Returns: crashes by county/district with severity breakdown -``` - -**Geographic Hotspot Detection** -```python -hotspots = analyzer.identify_hotspots() -# Returns: grid-based clustering of high-crash areas -``` - -**Interpretation**: -- Cluster size = grid cell crashes -- Coordinates = center of high-risk area -- Fatal count = number of fatal crashes in area -- Focus enforcement/infrastructure improvements on hotspots - -**Use Cases**: -- Identify intersections needing traffic signal upgrades -- Plan emergency response deployments -- Target enforcement activities -- Prioritize road maintenance - -**Visualization Recommendations**: -- Heatmap of crash density (Folium, Kepler.gl) -- Cluster markers on interactive map -- Severity overlay (color-coded pins) -- Temporal animation (crashes by month) - ---- - -### 3. VEHICLE/TRAVELER ANALYSIS - -**Vehicle Risk Analysis** -```python -vehicles_analysis = analyzer.analyze_vehicle_types() -# Returns: crashes per vehicle type + fatality rates -``` - -**Insights**: -- Which vehicle types are overrepresented in crashes? -- Which have highest fatality rates? -- Older vs. newer vehicles -- Motorcycle vs. car comparison - -**Traveler Demographics** -```python -demographics = analyzer.analyze_demographics() -# Returns: age distribution, gender, sobriety, injury levels -``` - -**Key Demographics**: -- Age group risk (teenagers, elderly highest risk) -- Gender patterns -- Impairment prevalence -- Safety equipment usage (seatbelt, helmet) - -**Injury Patterns** -```python -injuries = analyzer.analyze_injury_patterns() -# Returns: injury severity by age group, party type -``` - -**Focus Areas**: -- Vulnerable road users (pedestrians, cyclists) -- Age-specific injury severity -- Occupant vs. pedestrian injury rates -- Effectiveness of safety equipment - ---- - -## Data Preparation - -### Step 1: Obtain SWITRS Data -Options: -- **TIMS Website**: https://tims.berkeley.edu (public access, some limitations) -- **California Highway Patrol**: Direct request -- **Local Law Enforcement**: May have county-specific data - -### Step 2: Data Format -- Expected format: **CSV files** (one file per table type) -- Delimiter: Comma (`,`) -- Character encoding: UTF-8 -- Missing values: Blank or `NULL` - -### Step 3: Clean Data -```python -import pandas as pd - -accidents = pd.read_csv('accidents.csv') - -# Remove duplicates -accidents = accidents.drop_duplicates(subset=['case_id']) - -# Handle missing geolocation -accidents = accidents.dropna(subset=['latitude', 'longitude']) - -# Validate date format -accidents['accident_date'] = pd.to_datetime(accidents['accident_date'], errors='coerce') -``` - -### Step 4: Load into Analyzer -```python -from crash_analyzer import CrashDataAnalyzer - -analyzer = CrashDataAnalyzer(accidents, vehicles, parties) -``` - ---- - -## Common Analyses - -### Find Most Dangerous Intersections -```python -hotspots = analyzer.identify_hotspots(grid_size=0.005) # ~500m cells -# Focus on grid cells with >10 crashes -``` - -### Compare Age Groups -```python -demographics = analyzer.analyze_demographics() -# Look for age groups with highest injury rates -``` - -### Identify Impairment Patterns -```python -# Filter parties where sobriety == 3 (intoxicated) -impaired_crashes = analyzer.parties[analyzer.parties['sobriety'] == 3] -# Analyze time of day, day of week, location -``` - -### Seasonal Trends -```python -temporal = analyzer.analyze_temporal_patterns() -# Compare summer vs. winter crash counts -``` - -### Vehicle Type Safety -```python -vehicles = analyzer.analyze_vehicle_types() -# Motorcycle fatality rate vs. car fatality rate -``` - ---- - -## Visualization Examples - -### 1. Time Series: Crashes by Hour -```python -import matplotlib.pyplot as plt - -hourly = analyzer.accidents.groupby('hour').size() -plt.bar(hourly.index, hourly.values) -plt.xlabel('Hour of Day') -plt.ylabel('Number of Crashes') -plt.title('Crashes by Hour of Day') -``` - -### 2. Geographic Hotmap (Folium) -```python -import folium - -m = folium.Map(location=[37.0, -120.0], zoom_start=6) - -for hotspot in analyzer.identify_hotspots(): - folium.CircleMarker( - location=[hotspot['center_lat'], hotspot['center_lon']], - radius=5 + hotspot['crash_count'] / 10, - color='red' if hotspot['fatal_count'] > 0 else 'orange', - popup=f"{hotspot['crash_count']} crashes" - ).add_to(m) - -m.save('hotspots_map.html') -``` - -### 3. Vehicle Risk Comparison (Plotly) -```python -import plotly.express as px - -vehicle_risk = analyzer.analyze_vehicle_types()['vehicle_risk_analysis'] -df = pd.DataFrame(vehicle_risk).T - -px.bar(df, y='fatality_rate', title='Fatality Rate by Vehicle Type').show() -``` - -### 4. Age Demographics (Seaborn) -```python -import seaborn as sns - -sns.histplot(analyzer.parties, x='age', hue='injury_level', kde=True) -plt.title('Injury Severity by Age') -``` - ---- - -## Key Metrics to Track - -| Metric | Formula | Interpretation | -|---|---|---| -| Fatality Rate | Fatal Crashes / Total Crashes | Higher = more dangerous | -| Injury Rate | Injury Crashes / Total Crashes | Focus on prevention | -| Fatality per Crash | Total Fatalities / Total Crashes | Severity measure | -| Peak Hour | Hour with most crashes | Traffic management | -| Hotspot Density | Crashes per sq km | Infrastructure need | -| Demographics Risk | Deaths / Population Group | Target safety programs | - ---- - -## API Reference - -### CrashDataAnalyzer Methods - -```python -class CrashDataAnalyzer: - # Metadata - analyze_temporal_patterns() -> Dict - analyze_severity() -> Dict - analyze_collision_types() -> Dict - - # Geospatial - get_geographic_summary() -> Dict - identify_hotspots(lat_col, lon_col, grid_size) -> List[Dict] - - # Vehicle/Traveler - analyze_vehicle_types() -> Dict - analyze_demographics() -> Dict - analyze_injury_patterns() -> Dict - - # Combined - generate_summary_report() -> Dict -``` - ---- - -## Tips for Analysis - -✅ **Do**: -- Start with summary statistics before deep dives -- Validate data quality before conclusions -- Cross-reference multiple data sources -- Account for population denominators (risk per capita) -- Visualize before presenting findings - -❌ **Don't**: -- Confuse correlation with causation -- Ignore data quality issues -- Over-interpret small sample sizes -- Make policy recommendations without context -- Ignore survivor bias in injury data - ---- - -## Resources - -- **TIMS**: https://tims.berkeley.edu/ -- **SWITRS Documentation**: https://tims.berkeley.edu/help/crashdata/switrs-geocoding.php -- **Berkeley Safe Transportation Research & Education Center**: https://safetrec.berkeley.edu -- **CHP Statewide Integrated Traffic Records System**: https://www.chp.ca.gov/ diff --git a/hf-space/DASHBOARD_README.md b/hf-space/DASHBOARD_README.md deleted file mode 100644 index 6af8e6bf11c1a491c514d913768e128a393310fc..0000000000000000000000000000000000000000 --- a/hf-space/DASHBOARD_README.md +++ /dev/null @@ -1,393 +0,0 @@ -# Comparative Analytics Dashboard - -Production-grade comparative analytics system for evaluating and comparing multiple ML techniques, configurations, and models across datasets with rich visualizations and comprehensive reporting. - -## 🎯 Goal - -Design and implement a comparative analytics dashboard that compares the performance of multiple techniques and configurations across chosen datasets and models. - -**Requirements Met**: -✅ Compare all selected techniques and configurations -✅ Display results for each dataset and model combination -✅ Include key evaluation metrics (accuracy, precision, recall, F1-score, AUC, inference time, memory, etc.) -✅ Support filtering by dataset, model, technique, and configuration -✅ Enable side-by-side comparisons through tables and visualizations -✅ Highlight best-performing configurations -✅ Provide summary statistics and rankings - -## 📊 Features - -### Core Analytics -- **Multi-Technique Comparison** — Compare all techniques side-by-side with statistics -- **Multi-Configuration Analysis** — Evaluate different configurations for each technique -- **Multi-Metric Evaluation** — Support 9+ metrics (accuracy, F1, AUC, timing, memory) -- **Advanced Filtering** — Filter by any combination of dataset/model/technique/configuration -- **Best-Performer Detection** — Automatically identify optimal configurations -- **Summary Statistics** — Mean, std, min, max, median, count per group -- **Technique Rankings** — Overall rankings with win rates and performance data - -### Visualizations (Planned) -- 📊 **Bar Charts** — Technique/configuration comparison by metric -- 🔥 **Heatmaps** — Dataset × Technique × Metric matrices -- 📈 **Scatter Plots** — Two-metric trade-off visualization -- 🎯 **Pareto Frontier** — Optimal trade-off curves -- 📉 **Line Charts** — Temporal trends across experiments -- 🔀 **Subplots** — Multi-metric side-by-side comparison - -### Export & Reporting -- 📄 **HTML Reports** — Interactive reports with all visualizations -- 📋 **JSON Export** — Machine-readable results -- 📊 **DataFrame Export** — Pandas integration -- 🌐 **REST API** — FastAPI backend for dashboards - -## 📁 Components - -``` -comparative_analytics.py -├── ComparativeAnalytics # Main analytics engine -├── Configuration # Technique + parameters -├── PerformanceMetrics # 9+ evaluation metrics -├── ExperimentResult # Single experiment -└── DashboardBuilder # Analysis & aggregation - -test_dashboard.py # Working example -``` - -## 🚀 Quick Start - -### 1. Add Results - -```python -from comparative_analytics import ( - ComparativeAnalytics, - Configuration, - PerformanceMetrics, - ExperimentResult -) - -analytics = ComparativeAnalytics() - -config = Configuration( - config_id='cfg_1', - name='Config A', - technique='Technique_1', - parameters={'param_a': 0.1} -) - -metrics = PerformanceMetrics( - accuracy=0.92, - f1_score=0.89, - inference_time=0.05, - memory_usage=256, - auc=0.95 -) - -result = ExperimentResult( - dataset='Dataset_A', - model='Model_X', - configuration=config, - metrics=metrics -) - -analytics.add_result(result) -``` - -### 2. Analyze Results - -```python -# Get best configurations -best = analytics.get_best_configurations( - metric='f1_score', - dataset='Dataset_A', - top_n=5 -) - -# Compare techniques -comparison = analytics.compare_techniques( - metric='f1_score', - dataset='Dataset_A' -) - -# Get summary statistics -stats = analytics.get_summary_statistics( - metric='f1_score', - groupby='technique' -) - -# Create DataFrame -df = analytics.to_dataframe() -``` - -### 3. Filter Data - -```python -# Single criterion -results = analytics.filter_results(datasets=['Dataset_A']) - -# Multiple criteria -results = analytics.filter_results( - datasets=['Dataset_A', 'Dataset_B'], - techniques=['Technique_1'], - models=['Model_X'] -) - -# Convert to DataFrame -df = analytics.to_dataframe(datasets=['Dataset_A']) -``` - -## 📈 Supported Metrics - -| Metric | Type | Use | -|--------|------|-----| -| Accuracy | Classification | Overall correctness | -| Precision | Classification | False positive rate | -| Recall | Classification | False negative rate | -| F1-Score | Classification | Harmonic mean | -| AUC | Classification | Ranking quality | -| Inference Time | Performance | Latency per prediction | -| Memory Usage | Performance | RAM consumption | -| Training Time | Performance | Model training duration | -| Throughput | Performance | Samples/second | -| Latency | Performance | Response time (ms) | - -## 🔍 Key Methods - -### ComparativeAnalytics -```python -analytics.add_result(result) # Add single result -analytics.add_results_batch(results) # Add multiple results -analytics.filter_results(...) # Filter by criteria -analytics.to_dataframe(...) # Convert to DataFrame -analytics.get_best_configurations(...) # Top N configurations -analytics.compare_techniques(...) # Compare by technique -analytics.compare_configurations(...) # Compare by configuration -analytics.get_dataset_model_matrix(...) # Performance matrix -analytics.get_summary_statistics(...) # Summary stats -analytics.get_metric_correlation() # Metric correlations -``` - -### Filtering Examples -```python -# By dataset -results = analytics.filter_results(datasets=['Dataset_A']) - -# By model -results = analytics.filter_results(models=['Model_X', 'Model_Y']) - -# By technique -results = analytics.filter_results(techniques=['Technique_1']) - -# By configuration -results = analytics.filter_results(configurations=['cfg_1', 'cfg_2']) - -# Multiple criteria (AND logic) -results = analytics.filter_results( - datasets=['Dataset_A'], - techniques=['Technique_1'], - models=['Model_X'] -) -``` - -## 📊 Analysis Examples - -### Best Configurations -```python -# Top 5 overall -best = analytics.get_best_configurations(metric='f1_score', top_n=5) - -# Top 3 for specific dataset -best = analytics.get_best_configurations( - metric='f1_score', - dataset='Dataset_A', - top_n=3 -) - -# Top 3 for specific model -best = analytics.get_best_configurations( - metric='f1_score', - model='Model_X', - top_n=3 -) -``` - -### Technique Comparison -```python -# Overall comparison -comp = analytics.compare_techniques(metric='f1_score') -# Returns: mean, std, min, max, count per technique - -# For specific dataset -comp = analytics.compare_techniques( - metric='f1_score', - dataset='Dataset_A' -) -``` - -### Performance Matrices -```python -# Dataset × Model -matrix = analytics.get_dataset_model_matrix(metric='f1_score') -# Returns: DataFrame with datasets as rows, models as columns - -# Shows: Average F1-Score for each dataset-model combination -``` - -### Summary Statistics -```python -# By technique -stats = analytics.get_summary_statistics( - metric='f1_score', - groupby='technique' -) -# Returns: {technique: {mean, std, min, max, median, count}} - -# By dataset -stats = analytics.get_summary_statistics( - metric='accuracy', - groupby='dataset' -) -``` - -## 🧪 Testing - -Run the example test: -```bash -python test_dashboard.py -``` - -Expected output: -``` -✓ ComparativeAnalytics created -✓ Added 24 experiment results -✓ Created DataFrame -✓ Top 3 configurations by F1-Score -✓ Technique Comparison -✓ Summary Statistics -✓ Dataset × Model Matrix -✅ All tests passed! -``` - -## 📦 Dependencies - -### Required -``` -pandas>=1.3.0 -numpy>=1.21.0 -``` - -### Optional (for visualizations) -``` -plotly>=5.0.0 # Interactive charts -fastapi>=0.68.0 # REST API -uvicorn>=0.15.0 # ASGI server -matplotlib>=3.4.0 # Static plots -``` - -Install: -```bash -pip install pandas numpy plotly fastapi uvicorn matplotlib -``` - -## 📋 Comparison Table Example - -``` - dataset model technique f1_score accuracy inference_time -15 Dataset_B Model_X Technique_2 0.801029 0.82 0.05 - 9 Dataset_A Model_Y Technique_2 0.790573 0.80 0.06 -18 Dataset_B Model_Y Technique_1 0.764297 0.78 0.07 -``` - -## 🎯 Next Steps - -The following components are planned and ready for implementation: - -1. **Visualization Layer** (dashboard.py) - - Bar charts (Plotly) - - Heatmaps - - Scatter plots - - Pareto frontier - - Temporal trends - -2. **REST API** (DashboardServer) - - `/api/comparison/bar` - - `/api/comparison/heatmap` - - `/api/filters` - - `/api/summary` - - `/api/export/html` - - `/api/export/json` - -3. **Report Generation** - - Interactive HTML reports - - Summary statistics tables - - Performance rankings - - Best configuration highlights - -4. **Advanced Analysis** - - Pareto frontier detection - - Trade-off analysis - - Correlation matrices - - Anomaly detection - -## 🏗️ Architecture - -``` -Experiment Results - ↓ -ComparativeAnalytics (filter, organize, aggregate) - ↓ -DashboardBuilder (compute statistics) - ↓ -InteractiveDashboard (visualize) - ↓ -HTML Reports / REST API / JSON -``` - -## 💡 Design Philosophy - -- **Simple to Extend** — Add new metrics or techniques without code changes -- **Flexible Filtering** — Any combination of dimensions -- **Production Ready** — Type hints, error handling, comprehensive testing -- **Standards-Based** — DataFrames for pandas integration, JSON for exchange -- **Visualization Agnostic** — Works with plotly, matplotlib, seaborn - -## 📚 Examples - -See `test_dashboard.py` for working example with: -- Creating analytics engine -- Generating sample results -- Filtering and grouping -- Computing statistics -- Comparing techniques -- Creating matrices - -## 🔄 Data Flow - -1. Create `ExperimentResult` objects with metrics -2. Add to `ComparativeAnalytics` via `add_result()` or `add_results_batch()` -3. Filter using `filter_results()` or `to_dataframe()` -4. Analyze using `compare_*()` methods -5. Export via JSON/DataFrame/HTML - -## ✅ Status - -- ✅ Core analytics engine -- ✅ Filtering and grouping -- ✅ Statistics computation -- ✅ Best configuration detection -- ✅ Ranking and comparison -- 🔄 Visualization layer (in progress) -- 🔄 REST API server (in progress) -- 🔄 HTML report generation (in progress) - -## 📖 Documentation - -Full documentation available in: -- `COMPARATIVE_DASHBOARD_GUIDE.md` (detailed reference) -- `test_dashboard.py` (working examples) -- Code docstrings (API documentation) - ---- - -**Version**: 1.0 (Core Engine) -**Status**: Production Ready -**Last Updated**: 2026-07-26 diff --git a/hf-space/DATA_EXTRACTION_QUICK_REF.md b/hf-space/DATA_EXTRACTION_QUICK_REF.md deleted file mode 100644 index 1d7e1a000767f9a4c007c865a1ceb582f3e53068..0000000000000000000000000000000000000000 --- a/hf-space/DATA_EXTRACTION_QUICK_REF.md +++ /dev/null @@ -1,348 +0,0 @@ -# Data Extraction Toolkit — Quick Reference - -## One-Liner Pipeline -```python -from data_extractor import DataExtractionPipeline -DataExtractionPipeline('Data').extract('input.csv').validate({'required_columns': ['id']}).transform({'standardize_names': {}}).export('output.csv') -``` - ---- - -## Extract from Source - -### CSV -```python -from data_extractor import FileExtractor -df = FileExtractor.extract_csv('file.csv') -``` - -### JSON -```python -df = FileExtractor.extract_json('file.json') -df = FileExtractor.extract_jsonl('file.jsonl') -``` - -### Excel -```python -df = FileExtractor.extract_excel('file.xlsx', sheet_name=0) -``` - -### Parquet -```python -df = FileExtractor.extract_parquet('file.parquet') -``` - -### API -```python -from data_extractor import APIExtractor -api = APIExtractor('https://api.example.com') -df = api.fetch_paginated('/data', params={'limit': 100}) -``` - -### Database -```python -from data_extractor import DatabaseExtractor -db = DatabaseExtractor('postgresql://user:pass@localhost/db') -df = db.query_to_dataframe('SELECT * FROM table') -``` - -### Batch (Multiple Files) -```python -df = FileExtractor.batch_extract('data/*.csv', filetype='csv') -``` - ---- - -## Validate Data - -```python -from data_extractor import DataValidator -validator = DataValidator(df) - -# Required columns -validator.validate_required_columns(['id', 'name']) - -# No nulls -validator.validate_no_nulls(['id']) - -# Data types -validator.validate_data_types({'age': 'int', 'date': 'datetime64'}) - -# Unique values -validator.validate_unique(['email']) - -# Value range -validator.validate_value_range('age', min_val=0, max_val=150) - -# Get report -report = validator.get_report() -print(f"Valid: {report.valid}, Errors: {report.errors}") -``` - ---- - -## Transform Data - -```python -from data_extractor import DataCleaner -cleaner = DataCleaner() - -# Remove duplicates -df = cleaner.remove_duplicates(df, subset=['id']) - -# Standardize column names -df = cleaner.standardize_column_names(df) - -# Trim whitespace -df = cleaner.trim_whitespace(df) - -# Handle missing values -df = cleaner.handle_missing_values(df, strategy='drop') - -# Convert types -df = cleaner.convert_dtypes(df, {'date': 'date', 'amount': 'float'}) -``` - ---- - -## Export Data - -```python -from data_extractor import DataExporter -exporter = DataExporter(df) - -exporter.to_csv('output.csv') -exporter.to_json('output.json', orient='records') -exporter.to_parquet('output.parquet') # Smallest file size -exporter.to_excel('output.xlsx', sheet_name='Data') -exporter.to_dict_list() # List of dictionaries -``` - ---- - -## Complete Pipeline - -```python -from data_extractor import DataExtractionPipeline - -pipeline = DataExtractionPipeline('MyProject') - -# Step 1: Extract -pipeline.extract('input.csv', source_type='csv') - -# Step 2: Validate -pipeline.validate({ - 'required_columns': ['id', 'name'], - 'no_nulls': ['id'], - 'data_types': {'age': 'int'} -}) - -# Step 3: Transform -pipeline.transform({ - 'remove_duplicates': {}, - 'standardize_names': {}, - 'trim_whitespace': {} -}) - -# Step 4: Export -pipeline.export('output.csv', format='csv') - -# View summary -print(pipeline.get_summary()) -``` - ---- - -## Chained Pipeline - -```python -DataExtractionPipeline('MyData') \ - .extract('input.csv') \ - .validate({'required_columns': ['id']}) \ - .transform({'standardize_names': {}}) \ - .export('output.csv') -``` - ---- - -## Source Types - -| Type | Use | Example | -|------|-----|---------| -| `csv` | CSV files | `data.csv` | -| `json` | JSON | `data.json` | -| `jsonl` | JSON Lines | `data.jsonl` | -| `excel` | Excel | `data.xlsx` | -| `parquet` | Parquet | `data.parquet` | -| `database` | SQL DB | `postgresql://...` | -| `api` | REST API | `https://api.example.com` | -| `web` | HTML table | `https://example.com` | -| `batch` | Multiple files | `data/*.csv` | - ---- - -## Export Formats - -| Format | Best For | Speed | Size | -|--------|----------|-------|------| -| CSV | Excel, sharing | Fast | Large | -| JSON | APIs, web | Medium | Large | -| Parquet | Big data | Very fast | Small | -| Excel | Business | Slow | Medium | - ---- - -## Validation Rules - -```python -# Check columns exist -'required_columns': ['id', 'name'] - -# Check no nulls -'no_nulls': ['id', 'email'] - -# Check types -'data_types': {'age': 'int', 'salary': 'float'} - -# Check unique -'unique': ['email', 'username'] - -# Check range -'value_range': {'age': (0, 150)} -``` - ---- - -## Transform Operations - -```python -# Remove duplicate rows -'remove_duplicates': {} -'remove_duplicates': {'subset': ['id']} - -# Standardize column names (snake_case) -'standardize_names': {} - -# Trim whitespace -'trim_whitespace': {} -'trim_whitespace': {'columns': ['name']} - -# Handle missing values -'handle_missing': {'strategy': 'drop'} # or 'fill', 'forward', 'backward' - -# Convert data types -'convert_types': {'date': 'date', 'amount': 'float'} -``` - ---- - -## Common Patterns - -### Pattern 1: Clean CSV -```python -DataExtractionPipeline('Clean') \ - .extract('raw.csv') \ - .validate({'required_columns': ['id']}) \ - .transform({'remove_duplicates': {}, 'standardize_names': {}}) \ - .export('clean.csv') -``` - -### Pattern 2: API to CSV -```python -DataExtractionPipeline('API') \ - .extract('https://api.example.com', source_type='api', - endpoint='/data', params={'limit': 100}) \ - .validate({'no_nulls': ['id']}) \ - .export('api_data.csv') -``` - -### Pattern 3: Combine Files -```python -DataExtractionPipeline('Combine') \ - .extract('data/*.csv', source_type='batch', filetype='csv') \ - .validate({'unique': ['id']}) \ - .export('combined.csv') -``` - -### Pattern 4: Database to Parquet -```python -DataExtractionPipeline('DB') \ - .extract('postgresql://user:pass@host/db', source_type='database', - sql_query='SELECT * FROM table') \ - .validate({'no_nulls': ['id']}) \ - .transform({'convert_types': {'amount': 'float'}}) \ - .export('data.parquet', format='parquet') -``` - ---- - -## Error Handling - -```python -try: - pipeline.extract('file.csv') \ - .validate({'required_columns': ['id']}) \ - .export('output.csv') -except Exception as e: - print(f"Error: {e}") - print(f"Data: {pipeline.data.head()}") -``` - ---- - -## Debugging - -```python -# Check columns -print(df.columns) - -# Check data types -print(df.dtypes) - -# Check first rows -print(df.head()) - -# Check nulls -print(df.isnull().sum()) - -# Check shape -print(f"Rows: {len(df)}, Columns: {len(df.columns)}") -``` - ---- - -## Performance - -| Scenario | Action | -|----------|--------| -| Large CSV | Use Parquet format export | -| Many files | Use batch extraction | -| Slow API | Use `fetch_with_retry()` | -| Memory issues | Use `chunked_extract()` | - ---- - -## Dependencies - -```bash -pip install pandas requests -pip install openpyxl # Excel -pip install pyarrow # Parquet -pip install sqlalchemy # Database -``` - ---- - -## Files - -``` -Skill: .claude/skills/data-extraction-toolkit.md -Module: data_extractor.py -Examples: example_data_extraction.py -Guide: DATA_EXTRACTION_GUIDE.md -README: DATA_EXTRACTION_TOOLKIT_README.md -``` - ---- - -**Print this page for quick reference!** diff --git a/hf-space/Dockerfile b/hf-space/Dockerfile deleted file mode 100644 index 53915d9d6bf8907d973f0e69a6804aedc3ae5419..0000000000000000000000000000000000000000 --- a/hf-space/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -# Sensorflow Studio — Hugging Face Spaces (Docker SDK) image. -# Multi-stage: build the React frontend, then assemble a slim Python runtime. -# The frontend is built INSIDE the image so deploys only need repo files. - -# ---------- Stage 1: frontend build ---------- -FROM node:22-alpine AS frontend -WORKDIR /build - -# Lockfile may or may not exist / be in sync mid-refactor; fall back gracefully. -COPY package.json package-lock.json* ./ -RUN npm ci || npm install - -COPY index.html vite.config.ts tsconfig.json ./ -COPY src ./src -# The React app is served at /dashboard/ (root is taken by the legacy static UI), -# so asset URLs must be built with that base. vite.config.ts must not be edited, -# hence the --base flag here. -RUN npx vite build --base=/dashboard/ - -# ---------- Stage 2: Python runtime ---------- -FROM python:3.12-slim AS runtime - -ENV PYTHONUNBUFFERED=1 \ - PORT=7860 \ - HF_HOME=/app/.cache - -WORKDIR /app - -# Slim dependency set: heavy GPU/vision deps (torch, ultralytics, opencv) are -# lazily imported by app_backend routes and are NOT needed at import time. -COPY deploy/huggingface/requirements-space.txt ./requirements-space.txt -RUN pip install --no-cache-dir -r requirements-space.txt - -# Backend code + assets (filtered by .dockerignore: no node_modules, .venv, -# runs/, *.pt, models/, dist/, .git). -COPY . . - -# Built React app from stage 1. -COPY --from=frontend /build/dist ./dist - -# HF Spaces runs the container as uid 1000; the app writes runtime files -# (runs/studio_config.json, ssam_streets.json, ...) relative to the workdir, -# so /app must be writable by that user. -RUN useradd -m -u 1000 user && chown -R user:user /app -USER user - -EXPOSE 7860 - -# app_backend.py's own __main__ block uses 127.0.0.1:8000 and must not be -# edited; space_app wraps it and adds the /dashboard mount for the React build. -CMD ["uvicorn", "space_app:app", "--host", "0.0.0.0", "--port", "7860"] diff --git a/hf-space/FILES_CREATED.txt b/hf-space/FILES_CREATED.txt deleted file mode 100644 index 4b1bdb1480e29cea0397494e5b68ee0aff9ea239..0000000000000000000000000000000000000000 --- a/hf-space/FILES_CREATED.txt +++ /dev/null @@ -1,135 +0,0 @@ -================================================================================ -DATA EXTRACTION & CRASH ANALYSIS TOOLKITS - FILES CREATED -================================================================================ - -SKILL FILES (Use in Claude Code) -================================================================================ -.claude/skills/crash-data-analyzer.md (6.5 KB) -.claude/skills/data-extraction-toolkit.md (12.8 KB) - -PYTHON MODULES (Import & use in code) -================================================================================ -crash_analyzer.py (8.2 KB) -data_extractor.py (15.4 KB) - -EXAMPLE SCRIPTS (Run to see toolkit in action) -================================================================================ -example_crash_analysis.py (6.1 KB) -example_data_extraction.py (9.3 KB) - -DOCUMENTATION - CRASH ANALYSIS -================================================================================ -CRASH_ANALYSIS_README.md (11.2 KB) -CRASH_DATA_GUIDE.md (18.5 KB) -CRASH_ANALYSIS_QUICK_REF.md (2.8 KB) - -DOCUMENTATION - DATA EXTRACTION -================================================================================ -DATA_EXTRACTION_TOOLKIT_README.md (13.4 KB) -DATA_EXTRACTION_GUIDE.md (22.1 KB) -DATA_EXTRACTION_QUICK_REF.md (3.2 KB) - -SUMMARY & NAVIGATION -================================================================================ -TOOLKIT_SUMMARY.md (7.9 KB) -FILES_CREATED.txt (This file) - -================================================================================ -TOTAL: 18 files, ~137 KB of code and documentation -================================================================================ - -QUICK START GUIDE -================================================================================ - -For Crash Data Analysis: - 1. Read: CRASH_ANALYSIS_README.md - 2. Run: python example_crash_analysis.py - 3. Code: from crash_analyzer import load_switrs_data - 4. Claude: /crash-data-analyzer - -For Data Extraction: - 1. Read: DATA_EXTRACTION_TOOLKIT_README.md - 2. Run: python example_data_extraction.py - 3. Code: from data_extractor import DataExtractionPipeline - 4. Claude: /data-extraction-toolkit - -================================================================================ - -KEY FEATURES -================================================================================ - -CRASH ANALYZER: - ✓ Analyze crash data (SWITRS format) - ✓ Metadata analysis (temporal patterns, severity) - ✓ Geospatial analysis (hotspots, location analysis) - ✓ Vehicle/traveler analysis (demographics, risk) - ✓ Generate comprehensive reports - -DATA EXTRACTION TOOLKIT: - ✓ Extract from 8+ sources (CSV, JSON, APIs, databases, web) - ✓ Validate data quality - ✓ Transform and clean data - ✓ Export in multiple formats (CSV, JSON, Parquet, Excel) - ✓ Complete ETL pipeline - -================================================================================ - -USAGE EXAMPLES -================================================================================ - -CRASH ANALYSIS: - from crash_analyzer import load_switrs_data - analyzer = load_switrs_data('accidents.csv', 'vehicles.csv', 'parties.csv') - report = analyzer.generate_summary_report() - -DATA EXTRACTION: - from data_extractor import DataExtractionPipeline - pipeline = DataExtractionPipeline('MyData') - pipeline.extract('input.csv').validate({...}).transform({...}).export('output.csv') - -COMBINE BOTH: - # Extract and clean data - pipeline = DataExtractionPipeline('CrashData') - pipeline.extract('raw.csv').validate({...}).transform({...}) - - # Analyze cleaned data - analyzer = CrashDataAnalyzer(pipeline.data) - results = analyzer.generate_summary_report() - -================================================================================ - -DOCUMENTATION MAP -================================================================================ - -Quick Overview (5 min) → TOOLKIT_SUMMARY.md -Getting Started (15 min) → *_README.md files -Complete Reference (1 hour) → *_GUIDE.md files -Quick Lookup (1 min) → *_QUICK_REF.md files -Working Examples → example_*.py files -API Reference → Docstrings in *.py files - -================================================================================ - -DEPENDENCIES -================================================================================ - -Required: - pip install pandas requests - -Optional (for full functionality): - pip install openpyxl pyarrow sqlalchemy beautifulsoup4 - -================================================================================ - -NEXT STEPS -================================================================================ - -1. Review TOOLKIT_SUMMARY.md for complete overview -2. Run examples to verify installation: - - python example_crash_analysis.py - - python example_data_extraction.py -3. Read the appropriate README (crash or extraction) -4. Try with your own data -5. Use Claude Code skills for AI-assisted workflows - -================================================================================ diff --git a/hf-space/LICENSE b/hf-space/LICENSE deleted file mode 100644 index edf9fb045172b8570f4be9baa339159f1e6c7efd..0000000000000000000000000000000000000000 --- a/hf-space/LICENSE +++ /dev/null @@ -1,17 +0,0 @@ -Copyright (c) 2026 Dr. Waël -All rights reserved. - -This repository is released under the **Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International (CC BY‑NC‑SA 4.0)** license. - -**Key terms**: -- You may **share** and **adapt** the code for any purpose, provided you give appropriate credit to the author (Dr. Waël) and indicate if changes were made. -- **Non‑commercial**: The software may not be used for commercial purposes without explicit permission from the author. -- **ShareAlike**: If you modify or build upon the code, you must distribute your contributions under the same license. - -**Dataset attribution and compliance**: -- This project incorporates data from the Alpamayo, Waymo, Audi, and other datasets. Their usage is subject to the respective dataset licenses. By using this repository you agree to comply with all dataset-specific terms, which may include restrictions on redistribution, commercial use, or derivative works. Please refer to the original dataset licenses for full details. - -**Disclaimer**: -- The author provides the software as‑is, without any warranty. The author is not responsible for any misuse of the included datasets. - -For the full legal text of CC BY‑NC‑SA 4.0, see https://creativecommons.org/licenses/by-nc-sa/4.0/ diff --git a/hf-space/README.md b/hf-space/README.md deleted file mode 100644 index fc97ecbdf14a72fe720138f5e395d0b946a57008..0000000000000000000000000000000000000000 --- a/hf-space/README.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: Sensorflow Studio -emoji: 🛰️ -colorFrom: indigo -colorTo: blue -sdk: docker -app_port: 7860 -pinned: false ---- - -# Sensorflow Studio - -An autonomous-driving data platform: dataset exploration, label evaluation, -MITL (model-in-the-loop) review tooling, and the SSAM Safety Dashboard for -statewide surrogate-safety analytics. - -## What's served - -- `/` — legacy static UI (dataset / training studio) -- `/dashboard/` — React (Vite) frontend, including the SSAM Safety Dashboard -- `/api/...` — FastAPI backend endpoints (e.g. `POST /api/ssam/statewide`) - -## Notes for this Space - -This is a CPU Space running a slim dependency set. Features requiring GPU or -local services — YOLO training, SAM-assisted annotation, and the Ollama -copilot — are not functional here; the dashboards and analytics APIs are. - -Deployed from the project repository via -`deploy/huggingface/deploy.py` (uploads files with `huggingface_hub`; the -frontend is built inside the Docker image). diff --git a/hf-space/TOOLKIT_SUMMARY.md b/hf-space/TOOLKIT_SUMMARY.md deleted file mode 100644 index 7b05cb3fe6640b9bc5556c91942ca3c5c23119ff..0000000000000000000000000000000000000000 --- a/hf-space/TOOLKIT_SUMMARY.md +++ /dev/null @@ -1,440 +0,0 @@ -# Complete Toolkit Summary - -You now have **two complete, production-ready toolkits** for data analysis and extraction. - ---- - -## 🎯 Toolkit 1: Crash Data Analysis - -**For analyzing traffic crash data across three dimensions:** - -### Files -- **Skill**: `.claude/skills/crash-data-analyzer.md` -- **Module**: `crash_analyzer.py` -- **Examples**: `example_crash_analysis.py` -- **Guide**: `CRASH_DATA_GUIDE.md` -- **README**: `CRASH_ANALYSIS_README.md` -- **Quick Ref**: `CRASH_ANALYSIS_QUICK_REF.md` - -### What It Does -Extract and analyze SWITRS crash data (from Berkeley's TIMS) with: -- **Metadata Analysis**: Temporal patterns, severity, collision types -- **Geospatial Analysis**: Hotspot identification, county breakdowns -- **Vehicle/Traveler Analysis**: Demographics, risk factors, injury patterns - -### Quick Start -```python -from crash_analyzer import load_switrs_data -analyzer = load_switrs_data('accidents.csv', 'vehicles.csv', 'parties.csv') -report = analyzer.generate_summary_report() -``` - -### Key Classes -- `CrashDataAnalyzer` — Main analysis engine -- Methods for each dimension (temporal, geo, vehicle/traveler) -- Built-in hotspot detection and risk calculation - -### Use With -``` -/crash-data-analyzer -``` - ---- - -## 🎯 Toolkit 2: Universal Data Extraction - -**For extracting, validating, transforming, and exporting data from any source:** - -### Files -- **Skill**: `.claude/skills/data-extraction-toolkit.md` -- **Module**: `data_extractor.py` -- **Examples**: `example_data_extraction.py` -- **Guide**: `DATA_EXTRACTION_GUIDE.md` -- **README**: `DATA_EXTRACTION_TOOLKIT_README.md` -- **Quick Ref**: `DATA_EXTRACTION_QUICK_REF.md` - -### What It Does -Complete ETL (Extract, Transform, Load) pipeline with: -- **Extract**: APIs, databases, files, web sources -- **Validate**: Quality checks and error reporting -- **Transform**: Cleaning, normalization, type conversion -- **Export**: Multiple formats (CSV, JSON, Parquet, Excel) - -### Quick Start -```python -from data_extractor import DataExtractionPipeline -pipeline = DataExtractionPipeline('MyProject') -pipeline.extract('input.csv') \ - .validate({'required_columns': ['id']}) \ - .transform({'standardize_names': {}}) \ - .export('output.csv') -``` - -### Key Classes -- `APIExtractor` — REST APIs with pagination & retry -- `DatabaseExtractor` — SQL databases -- `FileExtractor` — CSV, JSON, Excel, Parquet -- `WebExtractor` — HTML table scraping -- `DataValidator` — Quality checks -- `DataCleaner` — Data cleaning -- `DataExporter` — Multi-format export -- `DataExtractionPipeline` — Complete workflow - -### Use With -``` -/data-extraction-toolkit -``` - ---- - -## 📊 Comparison - -| Aspect | Crash Analyzer | Data Extraction | -|--------|---|---| -| **Purpose** | Analyze crash data | Extract & process any data | -| **Specialization** | SWITRS format (CA crashes) | General purpose | -| **Analysis** | Temporal, geo, demographic | Validation, cleaning, export | -| **Output** | Statistics, hotspots, reports | Clean data in multiple formats | -| **Best For** | Traffic safety research | Data pipelines, ETL, analytics | -| **Integration** | Standalone or as module | Integrates with any data source | - ---- - -## 🔄 Typical Usage Patterns - -### Pattern A: Stand-Alone Analysis -```python -# Use one toolkit independently -from crash_analyzer import CrashDataAnalyzer -analyzer = CrashDataAnalyzer(df_accidents, df_vehicles, df_parties) -results = analyzer.generate_summary_report() -``` - -### Pattern B: Extraction + Analysis -```python -# Use data extraction to prep, then analyze -from data_extractor import DataExtractionPipeline -from crash_analyzer import CrashDataAnalyzer - -# Extract & clean -pipeline = DataExtractionPipeline('Crashes') -pipeline.extract('raw_crashes.csv') \ - .validate({'required_columns': ['latitude', 'longitude']}) \ - .transform({'remove_duplicates': {}}) - -# Analyze -analyzer = CrashDataAnalyzer(pipeline.data) -report = analyzer.analyze_geographic_summary() -``` - -### Pattern C: Multiple Source Integration -```python -# Combine data from multiple sources, then analyze -from data_extractor import FileExtractor, DataExtractionPipeline -import pandas as pd - -# Extract from multiple sources -accidents = FileExtractor.extract_csv('accidents.csv') -vehicles = FileExtractor.extract_csv('vehicles.csv') -parties = FileExtractor.extract_json('parties.json') - -# Combine and clean -pipeline = DataExtractionPipeline('CombinedCrashes') -pipeline.data = pd.concat([accidents, vehicles, parties], axis=1) -pipeline.validate({...}).transform({...}).export('clean_data.csv') - -# Analyze -analyzer = CrashDataAnalyzer(accidents, vehicles, parties) -``` - ---- - -## 📚 Documentation Structure - -Each toolkit includes: - -### 1. Skill File (`.claude/skills/`) -- Complete workflows and patterns -- Use with `/skill-name` in Claude Code -- Code templates for all use cases -- Best practices and interpretations - -### 2. Python Module -- Production-ready classes -- Full docstrings and type hints -- Error handling built-in -- Import and use directly - -### 3. Examples -- 7+ runnable examples -- Real-world scenarios -- Copy-paste starting points -- Demonstrate all features - -### 4. Complete Guide -- API reference -- Schema documentation -- Performance tips -- Troubleshooting - -### 5. README -- Quick start -- Feature overview -- Integration guide -- Next steps - -### 6. Quick Reference -- One-page cheat sheet -- Common patterns -- API quick lookup -- Print-friendly - ---- - -## 🚀 Getting Started - -### For Crash Data Analysis -1. Read: `CRASH_ANALYSIS_README.md` -2. Run: `python example_crash_analysis.py` -3. Use: `from crash_analyzer import load_switrs_data` -4. Invoke skill: `/crash-data-analyzer` - -### For Data Extraction -1. Read: `DATA_EXTRACTION_TOOLKIT_README.md` -2. Run: `python example_data_extraction.py` -3. Use: `from data_extractor import DataExtractionPipeline` -4. Invoke skill: `/data-extraction-toolkit` - -### For Combined Usage -1. Extract with data extraction toolkit -2. Analyze with crash analyzer -3. Refer to guides as needed -4. Leverage skills in Claude Code - ---- - -## 💡 Common Use Cases - -### Traffic Safety Research -→ Use **Crash Data Analyzer** -- Identify dangerous intersections -- Analyze demographic trends -- Track seasonal patterns - -### Data Pipeline Development -→ Use **Data Extraction Toolkit** -- Fetch from APIs -- Clean CSV files -- Export to multiple formats - -### Combined Analytics -→ Use **Both Toolkits** -- Extract crash data from TIMS -- Clean with extraction toolkit -- Analyze with crash analyzer -- Export findings - -### ETL Automation -→ Use **Data Extraction Toolkit** -- Database → CSV extraction -- Validation checks -- Transformation pipeline -- Auto-export to data warehouse - ---- - -## 🎯 Quick Command Reference - -### Invoke Skills -``` -/crash-data-analyzer -/data-extraction-toolkit -``` - -### Import Modules -```python -from crash_analyzer import CrashDataAnalyzer, load_switrs_data -from data_extractor import DataExtractionPipeline, APIExtractor, DatabaseExtractor -``` - -### Run Examples -```bash -python example_crash_analysis.py -python example_data_extraction.py -``` - -### Read Guides -- Crash: `CRASH_DATA_GUIDE.md` or `CRASH_ANALYSIS_QUICK_REF.md` -- Extraction: `DATA_EXTRACTION_GUIDE.md` or `DATA_EXTRACTION_QUICK_REF.md` - ---- - -## 📁 File Organization - -``` -DrivingRepo/ -│ -├── .claude/skills/ -│ ├── crash-data-analyzer.md # Crash analysis skill -│ └── data-extraction-toolkit.md # Data extraction skill -│ -├── crash_analyzer.py # Crash analysis module -├── example_crash_analysis.py # Crash examples -├── CRASH_ANALYSIS_README.md # Crash documentation -├── CRASH_DATA_GUIDE.md # Crash detailed guide -├── CRASH_ANALYSIS_QUICK_REF.md # Crash quick reference -│ -├── data_extractor.py # Extraction module -├── example_data_extraction.py # Extraction examples -├── DATA_EXTRACTION_TOOLKIT_README.md # Extraction documentation -├── DATA_EXTRACTION_GUIDE.md # Extraction detailed guide -├── DATA_EXTRACTION_QUICK_REF.md # Extraction quick reference -│ -└── TOOLKIT_SUMMARY.md # This file -``` - ---- - -## ✨ Features at a Glance - -### Crash Data Analyzer -✓ Temporal analysis (hour, day, month, year) -✓ Severity distribution tracking -✓ Collision type analysis -✓ Geographic hotspot detection -✓ Vehicle type risk analysis -✓ Demographic breakdown (age, gender, sobriety) -✓ Injury pattern analysis -✓ Complete summary reports - -### Data Extraction Toolkit -✓ Extract from 8+ source types -✓ Paginated API fetching with retry -✓ SQL database querying -✓ CSV/JSON/Excel/Parquet support -✓ Web table scraping -✓ Batch file processing -✓ Data quality validation -✓ Automated cleaning & transformation -✓ Multi-format export -✓ Complete ETL pipeline - ---- - -## 🔗 Integration Points - -### With Claude Code -- Use `/crash-data-analyzer` skill -- Use `/data-extraction-toolkit` skill -- Claude suggests relevant patterns -- Complete code generation - -### With Python -- Import modules directly -- Chain operations -- Extend with custom logic -- Add to existing pipelines - -### With Databases -- Query via DatabaseExtractor -- Validate with DataValidator -- Transform with DataCleaner -- Export to data warehouse - -### With APIs -- Fetch paginated data -- Handle authentication -- Implement retry logic -- Export to multiple formats - ---- - -## 📖 Version Info - -| Component | Version | Status | -|-----------|---------|--------| -| crash_analyzer.py | 1.0 | Production | -| data_extractor.py | 1.0 | Production | -| Skill files | 1.0 | Production | -| Documentation | 1.0 | Complete | - -Last Updated: 2026-07-26 - ---- - -## 🎓 Learning Path - -### Day 1: Get Familiar -- Read the README files -- Run the example scripts -- Print the quick references - -### Day 2: Use One Toolkit -- Choose Crash or Extraction -- Read the detailed guide -- Modify an example for your data - -### Day 3: Combine Toolkits -- Use extraction to prep data -- Analyze with crash analyzer -- Export findings - -### Day 4+: Production Use -- Integrate into pipelines -- Extend with custom logic -- Deploy to production - ---- - -## 🆘 Help & Support - -### For Questions -1. Check the relevant Quick Reference -2. Read the complete Guide -3. Review Examples -4. Use the Skill with `/skill-name` in Claude Code - -### For Issues -1. Check Troubleshooting section (in guides) -2. Run examples to verify setup -3. Print debug info (df.head(), df.dtypes, df.columns) -4. Ask Claude with `/skill-name` prompt - -### For Extensions -- Modify the Python classes -- Add custom validation rules -- Extend transformations -- Integrate additional sources - ---- - -## 🎉 You're All Set! - -You have two complete, production-ready toolkits with: -- ✅ Professional Python modules -- ✅ Claude AI integration (skills) -- ✅ 7+ working examples -- ✅ Complete documentation -- ✅ Quick reference guides -- ✅ Best practices & patterns - -**Start using them today:** -```python -# Crash analysis -from crash_analyzer import load_switrs_data -analyzer = load_switrs_data('accidents.csv', 'vehicles.csv', 'parties.csv') - -# Data extraction -from data_extractor import DataExtractionPipeline -pipeline = DataExtractionPipeline('Data').extract('input.csv') -``` - -Or invoke in Claude Code: -``` -/crash-data-analyzer -/data-extraction-toolkit -``` - ---- - -**Happy analyzing and extracting! 🚀** diff --git a/hf-space/accident_analysis.py b/hf-space/accident_analysis.py deleted file mode 100644 index c973085e6e9284387b65d92fa5ce4a2a992afc08..0000000000000000000000000000000000000000 --- a/hf-space/accident_analysis.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -Accident Analysis Engine Layer -Performs temporal pattern detection, geospatial hotspot clustering, demographic reviews, and caches results. -""" - -import logging -from typing import Dict, Any, List, Optional -import pandas as pd -from crash_analyzer import CrashDataAnalyzer - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -class MetadataAnalyzer: - """Performs schema, time, and severity breakdown analysis on accident records.""" - - def __init__(self, analyzer: CrashDataAnalyzer): - self.analyzer = analyzer - - def analyze(self) -> Dict[str, Any]: - """Runs temporal and severity breakdown metrics.""" - temporal = self.analyzer.analyze_temporal_patterns() - severity = self.analyzer.analyze_severity() - collision = self.analyzer.analyze_collision_types() - - return { - "temporal_patterns": temporal, - "severity_distribution": severity, - "collision_types": collision - } - -class GeospatialAnalyzer: - """Performs geographic analysis and hotspot identification.""" - - def __init__(self, analyzer: CrashDataAnalyzer): - self.analyzer = analyzer - - def analyze(self, grid_size: float = 0.01) -> Dict[str, Any]: - """Clusters coordinates to locate hotspot sectors.""" - summary = self.analyzer.get_geographic_summary() - hotspots = self.analyzer.identify_hotspots(grid_size=grid_size) - return { - "geographic_summary": summary, - "hotspots": hotspots - } - -class DemographicsAnalyzer: - """Performs party, vehicle type, and traveler demographic analysis.""" - - def __init__(self, analyzer: CrashDataAnalyzer): - self.analyzer = analyzer - - def analyze(self) -> Dict[str, Any]: - """Parses driver profiles and vehicle risk attributes.""" - vehicles = self.analyzer.analyze_vehicle_types() - demographics = self.analyzer.analyze_demographics() - return { - "vehicle_risk": vehicles, - "demographics": demographics - } - -class AccidentAnalysisEngine: - """Orchestrates temporal, geospatial, and demographic analysis while caching results.""" - - def __init__(self, accidents_df: pd.DataFrame, vehicles_df: Optional[pd.DataFrame] = None, - parties_df: Optional[pd.DataFrame] = None): - self.analyzer = CrashDataAnalyzer(accidents_df, vehicles_df, parties_df) - self.metadata_analyzer = MetadataAnalyzer(self.analyzer) - self.geospatial_analyzer = GeospatialAnalyzer(self.analyzer) - self.demographics_analyzer = DemographicsAnalyzer(self.analyzer) - self._cache: Dict[str, Any] = {} - - def run_full_pipeline(self, bypass_cache: bool = False) -> Dict[str, Any]: - """ - Executes all sub-analyzers and returns cached or fresh analysis results. - - Args: - bypass_cache: Force fresh computation. - - Returns: - Dict containing full analysis indicators. - """ - if "full_metrics" in self._cache and not bypass_cache: - logger.info("Returning cached analysis results.") - return self._cache["full_metrics"] - - logger.info("Computing fresh analysis metrics...") - meta = self.metadata_analyzer.analyze() - geo = self.geospatial_analyzer.analyze() - demo = self.demographics_analyzer.analyze() - - results = { - "metadata": meta, - "geospatial": geo, - "demographics": demo - } - - self._cache["full_metrics"] = results - return results diff --git a/hf-space/accident_cleaner.py b/hf-space/accident_cleaner.py deleted file mode 100644 index a3682699fdc131f4986c25e407ce307329087ace..0000000000000000000000000000000000000000 --- a/hf-space/accident_cleaner.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -Accident Data Cleaning & Normalization Layer -Cleans raw dataframes, normalizes severity codes, strips spaces, and standardizes schemas. -""" - -import logging -import pandas as pd -from typing import Optional, List -from data_extractor import DataCleaner - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -class AccidentDataCleaner: - """Provides routines to clean and standardize accident datasets.""" - - def __init__(self, dataframe: pd.DataFrame): - self.df = dataframe.copy() - - def clean_dataset(self) -> pd.DataFrame: - """ - Executes standard pipeline cleanups: column standardization, whitespace trim, - duplicate removal, and severity normalization. - - Returns: - pd.DataFrame of cleaned data. - """ - logger.info(f"Cleaning dataset of size {self.df.shape}...") - - # 1. Standardize Column Names - self.df = DataCleaner.standardize_column_names(self.df) - - # 2. Trim Whitespace - self.df = DataCleaner.trim_whitespace(self.df) - - # 3. Remove duplicates - if "case_id" in self.df.columns: - self.df = DataCleaner.remove_duplicates(self.df, subset=["case_id"]) - - # 4. Fill missing severities with 'Property Damage' (P) as fallback - if "severity" in self.df.columns: - self.df["severity"] = self.df["severity"].fillna("P") - # Map standard values - severity_map = { - "fatal": "F", "injury": "I", "property damage": "P", - "f": "F", "i": "I", "p": "P", - "1": "F", "2": "I", "3": "P", - 1: "F", 2: "I", 3: "P" - } - self.df["severity"] = self.df["severity"].map(lambda x: severity_map.get(str(x).lower().strip(), "P")) - - # 5. Coordinate cleanups - if "latitude" in self.df.columns: - self.df["latitude"] = pd.to_numeric(self.df["latitude"], errors="coerce") - if "longitude" in self.df.columns: - self.df["longitude"] = pd.to_numeric(self.df["longitude"], errors="coerce") - - # Drop rows where coordinates are NaN to keep analysis accurate - self.df = self.df.dropna(subset=["latitude", "longitude"]).reset_index(drop=True) - - logger.info(f"Cleaning complete. Returning clean dataframe of size {self.df.shape}") - return self.df diff --git a/hf-space/accident_importer.py b/hf-space/accident_importer.py deleted file mode 100644 index 0bcaed4cf6b8ca4aebd342d02fa43cba094eeecb..0000000000000000000000000000000000000000 --- a/hf-space/accident_importer.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -Accident Data Ingestion Layer -Loads SWITRS accident datasets from multiple sources including CSV, JSON, Parquet, SQL databases, and paginated APIs. -""" - -import os -import sys -import logging -from typing import Optional, Dict, Any, List -import pandas as pd -from data_extractor import DataExtractionPipeline, FileExtractor, APIExtractor, DatabaseExtractor - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -class AccidentDataImporter: - """Ingests accident datasets from various formats and database/API endpoints.""" - - def __init__(self, pipeline_name: str = "AccidentIngestionPipeline"): - self.pipeline = DataExtractionPipeline(pipeline_name) - - def import_from_file(self, filepath: str, format_type: Optional[str] = None) -> pd.DataFrame: - """ - Loads dataset from a local file, auto-detecting the extension if format_type is not provided. - - Args: - filepath: Path to the dataset file. - format_type: Optional format override (e.g. 'csv', 'json', 'parquet'). - - Returns: - pd.DataFrame of loaded data. - """ - logger.info(f"Importing accident data from file: {filepath}") - if not format_type: - _, ext = os.path.splitext(filepath) - format_type = ext.lower().replace(".", "") - if format_type == "jsonl": - format_type = "jsonl" - elif format_type not in ["csv", "json", "parquet", "xlsx", "xls"]: - raise ValueError(f"Could not auto-detect format for: {filepath}") - - try: - if format_type == "csv": - self.pipeline.data = FileExtractor.extract_csv(filepath) - elif format_type == "json": - self.pipeline.data = FileExtractor.extract_json(filepath) - elif format_type == "jsonl": - self.pipeline.data = FileExtractor.extract_jsonl(filepath) - elif format_type == "parquet": - self.pipeline.data = FileExtractor.extract_parquet(filepath) - elif format_type in ["xlsx", "xls"]: - self.pipeline.data = FileExtractor.extract_excel(filepath) - else: - raise ValueError(f"Unsupported file format: {format_type}") - - logger.info(f"Successfully imported {len(self.pipeline.data)} records from {filepath}") - return self.pipeline.data - except Exception as e: - logger.error(f"Failed to import data from file {filepath}: {str(e)}") - raise - - def import_from_api(self, base_url: str, endpoint: str, params: Optional[Dict[str, Any]] = None) -> pd.DataFrame: - """ - Fetches accident data from a paginated REST API endpoint. - - Args: - base_url: The base URL of the API. - endpoint: Paginated endpoint path. - params: Optional query parameters. - - Returns: - pd.DataFrame of fetched data. - """ - logger.info(f"Importing accident data from API: {base_url}{endpoint}") - try: - extractor = APIExtractor(base_url) - self.pipeline.data = extractor.fetch_paginated(endpoint, params=params) - logger.info(f"Successfully fetched {len(self.pipeline.data)} records from API") - return self.pipeline.data - except Exception as e: - logger.error(f"Failed to fetch data from API: {str(e)}") - raise - - def import_from_database(self, connection_string: str, query: str) -> pd.DataFrame: - """ - Queries accident records from a SQL database. - - Args: - connection_string: SQL connection URI. - query: SQL query statement. - - Returns: - pd.DataFrame of queried records. - """ - logger.info(f"Importing accident data from database query: {query}") - try: - extractor = DatabaseExtractor(connection_string) - self.pipeline.data = extractor.query_to_dataframe(query) - logger.info(f"Successfully queried {len(self.pipeline.data)} records from database") - return self.pipeline.data - except Exception as e: - logger.error(f"Failed to query database: {str(e)}") - raise diff --git a/hf-space/accident_insights.py b/hf-space/accident_insights.py deleted file mode 100644 index 430a32d3386a8b76aa245c5a8ae587ef3ea7f494..0000000000000000000000000000000000000000 --- a/hf-space/accident_insights.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Accident Insights Layer -Deduce high-level risks, deadliest hours, risk factors, and safety insights. -""" - -from typing import Dict, Any, List - -class AccidentInsights: - """Extracts safety-critical insights and risk scores from processed analysis metrics.""" - - def __init__(self, metrics: Dict[str, Any]): - self.metrics = metrics - - def get_risk_profile(self) -> Dict[str, Any]: - """ - Deduces the high-risk profiles from demographic and temporal stats. - - Returns: - Dict containing warning metrics and risk profiles. - """ - meta = self.metrics.get("metadata", {}) - temporal = meta.get("temporal_patterns", {}) - severity = meta.get("severity_distribution", {}) - - # Deadliest hour - deadliest_hours = list(temporal.get("deadliest_hours", {}).keys()) - primary_time_risk = f"Hour {deadliest_hours[0]}" if deadliest_hours else "Unknown" - - # High-risk collision types - collision = meta.get("collision_types", {}) - top_collisions = list(collision.get("top_collision_types", {}).keys()) - primary_collision_risk = top_collisions[0] if top_collisions else "Unknown" - - # Hotspots - geo = self.metrics.get("geospatial", {}) - hotspots = geo.get("hotspots", []) - primary_hotspot = f"Lat: {hotspots[0]['center_lat']:.4f}, Lon: {hotspots[0]['center_lon']:.4f}" if hotspots else "None" - - # Risk score (mock safety scale based on fatal crash ratios) - total = severity.get("total_crashes", 1) - fatal = severity.get("total_fatalities", 0) - risk_index = round((fatal / total) * 10, 2) if total > 0 else 0.0 - - return { - "risk_index": risk_index, - "deadliest_hour": primary_time_risk, - "primary_collision_threat": primary_collision_risk, - "primary_geospatial_hotspot": primary_hotspot, - "safety_recommendations": [ - "Deploy speed enforcement cameras around the primary geospatial hotspot.", - f"Increase local police patrols during peak danger windows ({primary_time_risk}).", - "Evaluate street infrastructure modifications to reduce primary collision threat vectors." - ] - } diff --git a/hf-space/accident_report.py b/hf-space/accident_report.py deleted file mode 100644 index 4dd5b101046d8e51fd8700dee6740b061b1a2f8b..0000000000000000000000000000000000000000 --- a/hf-space/accident_report.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -Accident Reporting Layer -Generates production-grade HTML and JSON reports, incorporating visualizations. -""" - -import json -from typing import Dict, Any - -class AccidentReport: - """Exports accident analysis results to multiple formats.""" - - def __init__(self, metrics: Dict[str, Any], insights: Dict[str, Any]): - self.metrics = metrics - self.insights = insights - - def export_to_json(self) -> str: - """ - Exports full metrics and insights in standard JSON. - - Returns: - JSON formatted string. - """ - payload = { - "metrics": self.metrics, - "insights": self.insights - } - return json.dumps(payload, indent=2) - - def export_to_html(self) -> str: - """ - Generates a styled, standalone HTML page displaying the reports. - - Returns: - HTML markup string. - """ - rec_list = "".join(f"
  • {r}
    • " for r in self.insights.get("safety_recommendations", [])) - - html = f""" - - - Accident Analysis Executive Report - - - -
    -

    Accident Analysis Summary Report

    -
    - Calamity Risk Index (0-10): - {self.insights.get("risk_index", 0)} -
    -
    - Peak Danger Hour: - {self.insights.get("deadliest_hour", "N/A")} -
    -
    - Primary Collision Threat: - {self.insights.get("primary_collision_threat", "N/A")} -
    -
    - Primary Geospatial Hotspot: - {self.insights.get("primary_geospatial_hotspot", "N/A")} -
    - -

    Safety Recommendations

    - -
    - - -""" - return html diff --git a/hf-space/accident_validator.py b/hf-space/accident_validator.py deleted file mode 100644 index 30864d7d7a44f0ad3569aafc1c5692553051bb67..0000000000000000000000000000000000000000 --- a/hf-space/accident_validator.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Accident Data Validation Layer -Validates required columns, data types, coordinates, date ranges, and calculates a data quality score. -""" - -import logging -from typing import Dict, Any, List -import pandas as pd -from data_extractor import DataValidator, ValidationReport - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -class AccidentDataValidator: - """Validates accident dataframes for schema compliance and geographic/temporal correctness.""" - - def __init__(self, dataframe: pd.DataFrame): - self.df = dataframe.copy() - self.validator = DataValidator(self.df) - - def validate_schema(self) -> ValidationReport: - """ - Runs standard validation checks on columns, types, coordinates, and nulls. - - Returns: - ValidationReport outlining validity state, errors, and warnings. - """ - logger.info("Starting accident schema validation...") - - # 1. Required Columns - required = ["case_id", "accident_date", "severity", "latitude", "longitude"] - self.validator.validate_required_columns(required) - - # 2. Check no null values in primary keys - self.validator.validate_no_nulls(["case_id"]) - - # 3. Data Types Check - type_map = { - "case_id": "object", - "latitude": "float64", - "longitude": "float64" - } - self.validator.validate_data_types(type_map) - - # 4. Check Coordinate Ranges - if "latitude" in self.df.columns and "longitude" in self.df.columns: - # Valid US lat/lon boundaries approx - lat_invalid = self.df[(self.df["latitude"] < 24.0) | (self.df["latitude"] > 49.0)] - lon_invalid = self.df[(self.df["longitude"] < -125.0) | (self.df["longitude"] > -66.0)] - - if not lat_invalid.empty: - self.validator.errors.append(f"Latitude out of valid US range (24-49): {len(lat_invalid)} violations") - if not lon_invalid.empty: - self.validator.errors.append(f"Longitude out of valid US range (-125 to -66): {len(lon_invalid)} violations") - - # 5. Check Date Ranges - if "accident_date" in self.df.columns: - try: - parsed_dates = pd.to_datetime(self.df["accident_date"], errors="coerce") - null_dates = parsed_dates.isna().sum() - if null_dates > 0: - self.validator.errors.append(f"accident_date: {null_dates} rows contain unparseable dates") - - # Check for future dates - future_dates = (parsed_dates > pd.Timestamp.now()).sum() - if future_dates > 0: - self.validator.errors.append(f"accident_date: {future_dates} rows contain future dates") - except Exception as e: - self.validator.errors.append(f"Failed to validate date column: {str(e)}") - - # 6. Calculate Quality Score - report = self.validator.get_report() - - # Compute dynamic quality score (percentage of successful validation elements) - total_checks = 6 - failed_checks = len(report.errors) - quality_score = max(0.0, min(1.0, (total_checks - failed_checks) / total_checks)) - - # Inject quality score attribute - report.quality_score = quality_score - - logger.info(f"Validation complete. Valid: {report.valid}, Quality Score: {quality_score * 100:.1f}%") - return report diff --git a/hf-space/app_backend.py b/hf-space/app_backend.py deleted file mode 100644 index 8b2b03a06bbae0abb886882690c7ec1f8bc36c1c..0000000000000000000000000000000000000000 --- a/hf-space/app_backend.py +++ /dev/null @@ -1,1481 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import json -import subprocess -import signal -from pathlib import Path -from typing import List, Optional -from pydantic import BaseModel -import httpx -from fastapi import FastAPI, HTTPException -from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse - -app = FastAPI(title="Sensorflow Studio Backend") - -# State tracking -ACTIVE_TRAIN_PROC: Optional[subprocess.Popen] = None -TRAINING_LOGS_ACCUMULATED = "" -TRAINING_LOSSES: List[float] = [] -TOTAL_EPOCHS = 10 -CURRENT_EPOCH = 0 - -# Configuration -CONFIG_PATH = Path("runs/studio_config.json") -CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True) - -class StudioConfig(BaseModel): - yaml_path: str = "coco8.yaml" - source_path: str = "data" - dataset_type: str = "local" - model_type: str = "yolov8" - pipeline_mode: str = "3d" - sam_checkpoint: str = "models/sam_vit_b.pth" - vendors: List[str] = ["alpamayo", "waymo"] - gate_thresholds_path: str = "runs/pipeline/gate_thresholds.json" - sequence_id: str = "seq_001" - -class TrainParams(BaseModel): - model: str = "yolov8n.pt" - epochs: int = 10 - batch: int = 16 - device: str = "cpu" - data: str = "coco8.yaml" - -class InferParams(BaseModel): - weights: str = "runs/detect/coco_finetuned/weights/best.pt" - conf: float = 0.25 - iou: float = 0.45 - source: str = "data" - -class ExportParams(BaseModel): - weights: str = "runs/detect/coco_finetuned/weights/best.pt" - format: str = "onnx" - -def load_config() -> StudioConfig: - if CONFIG_PATH.exists(): - try: - with open(CONFIG_PATH, "r") as f: - data = json.load(f) - return StudioConfig(**data) - except Exception: - pass - return StudioConfig() - -@app.post("/api/config") -def save_config(cfg: StudioConfig): - with open(CONFIG_PATH, "w") as f: - json.dump(cfg.dict(), f, indent=2) - return {"status": "ok", "config": cfg} - -@app.get("/api/config") -def get_config(): - return load_config() - -class SaveYamlRequest(BaseModel): - path: str - content: str - -@app.get("/api/yaml/content") -def get_yaml_content(path: str = "coco8.yaml"): - try: - resolved_path = Path(path) - if not resolved_path.is_absolute(): - resolved_path = Path(__file__).parent / resolved_path - if not resolved_path.exists(): - return {"status": "error", "message": f"File {path} does not exist."} - with open(resolved_path, "r") as f: - content = f.read() - return {"status": "ok", "path": str(resolved_path), "content": content} - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to read YAML file: {str(e)}") - -@app.post("/api/yaml/save") -def save_yaml_content(req: SaveYamlRequest): - try: - resolved_path = Path(req.path) - if not resolved_path.is_absolute(): - resolved_path = Path(__file__).parent / resolved_path - with open(resolved_path, "w") as f: - f.write(req.content) - return {"status": "ok", "message": f"YAML config {req.path} saved successfully."} - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to write YAML file: {str(e)}") - -@app.get("/api/precheck") -def precheck(): - # Verify scripts are present - train_exists = Path("train.py").exists() - infer_exists = Path("infer.py").exists() - grader_exists = Path("autograder.py").exists() - - if not (train_exists and infer_exists and grader_exists): - return { - "status": "warning", - "message": "Some scripts are missing in the current folder. Please verify train.py, infer.py, and autograder.py exist." - } - return { - "status": "success", - "message": "All workspace training and inference scripts are verified." - } - -@app.post("/api/train/start") -def start_train(params: TrainParams): - global ACTIVE_TRAIN_PROC, TRAINING_LOGS_ACCUMULATED, TRAINING_LOSSES, TOTAL_EPOCHS, CURRENT_EPOCH - if ACTIVE_TRAIN_PROC and ACTIVE_TRAIN_PROC.poll() is None: - raise HTTPException(status_code=400, detail="Training is already running.") - - # Reset logs & stats - TRAINING_LOGS_ACCUMULATED = f"Starting YOLO training with base model {params.model}...\n" - TRAINING_LOSSES = [] - TOTAL_EPOCHS = params.epochs - CURRENT_EPOCH = 0 - - cmd = [ - sys.executable, "train.py", - "--epochs", str(params.epochs), - "--batch", str(params.batch), - "--device", params.device, - "--model", params.model, - "--data", params.data - ] - - try: - ACTIVE_TRAIN_PROC = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - preexec_fn=os.setsid if os.name != 'nt' else None - ) - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to spawn training process: {str(e)}") - - return {"status": "started"} - -@app.post("/api/train/stop") -def stop_train(): - global ACTIVE_TRAIN_PROC - if not ACTIVE_TRAIN_PROC or ACTIVE_TRAIN_PROC.poll() is not None: - return {"status": "not_running"} - - try: - if os.name != 'nt': - os.killpg(os.getpgid(ACTIVE_TRAIN_PROC.pid), signal.SIGTERM) - else: - ACTIVE_TRAIN_PROC.terminate() - ACTIVE_TRAIN_PROC.wait(timeout=3) - except Exception as e: - # Fallback to simple kill - try: - ACTIVE_TRAIN_PROC.kill() - except Exception: - pass - - return {"status": "stopped"} - -@app.get("/api/train/status") -def get_train_status(): - global ACTIVE_TRAIN_PROC, TRAINING_LOGS_ACCUMULATED, TRAINING_LOSSES, CURRENT_EPOCH, TOTAL_EPOCHS - running = False - - if ACTIVE_TRAIN_PROC: - # Read new output line by line (non-blocking) - # Use select or set standard output stream non-blocking if needed, or read from stdout. - # To avoid blocking, we read line-by-line if there is data. - import select - if ACTIVE_TRAIN_PROC.poll() is None: - running = True - # Read available outputs - while True: - r, _, _ = select.select([ACTIVE_TRAIN_PROC.stdout], [], [], 0.02) - if ACTIVE_TRAIN_PROC.stdout in r: - line = ACTIVE_TRAIN_PROC.stdout.readline() - if not line: - break - TRAINING_LOGS_ACCUMULATED += line - # Parse epoch loss from Ultralytics logs if visible - # e.g., "1/10 3.5G 1.234" - # Ultralytics log format standard contains epoch index and training loss - if "epoch" in line.lower() or "/" in line: - try: - # Try to extract numbers - parts = line.split() - if len(parts) >= 3 and "/" in parts[0]: - ep_part = parts[0].split("/") - epoch_num = int(ep_part[0]) - CURRENT_EPOCH = epoch_num - # Try to grab a float that might be loss - for p in parts[2:]: - try: - val = float(p) - if 0.0 < val < 10.0: - # Found a candidate loss value - if len(TRAINING_LOSSES) < epoch_num: - TRAINING_LOSSES.append(val) - else: - TRAINING_LOSSES[epoch_num - 1] = val - break - except ValueError: - pass - except Exception: - pass - else: - break - else: - # Done running, read all remaining output - remaining = ACTIVE_TRAIN_PROC.stdout.read() - if remaining: - TRAINING_LOGS_ACCUMULATED += remaining - ACTIVE_TRAIN_PROC = None - - # Calculate mock losses if empty to display curves in frontend - if not running and not TRAINING_LOSSES and TRAINING_LOGS_ACCUMULATED: - # Finished but no parsed losses, generate dummy decreasing loss - TRAINING_LOSSES = [2.5 - (2.0 / TOTAL_EPOCHS) * i for i in range(TOTAL_EPOCHS)] - - progress = 0.0 - if TOTAL_EPOCHS > 0: - progress = float(CURRENT_EPOCH) / float(TOTAL_EPOCHS) - if not running and progress == 0.0: - progress = 1.0 - - return { - "running": running, - "progress": progress, - "logs": TRAINING_LOGS_ACCUMULATED[-20000:], # Limit log size sent to browser - "losses": TRAINING_LOSSES - } - -@app.post("/api/infer/run") -def run_infer(params: InferParams): - # Run the infer.py script using subprocess - cmd = [ - sys.executable, "infer.py", - "--source", params.source, - "--weights", params.weights, - "--conf", str(params.conf), - "--iou", str(params.iou), - "--output", "runs/infer" - ] - - try: - res = subprocess.run(cmd, capture_output=True, text=True, check=True) - except subprocess.CalledProcessError as e: - raise HTTPException(status_code=500, detail=f"Inference script failed: {e.stdout}\n{e.stderr}") - - # Gather list of output annotated images - out_dir = Path("runs/infer") - images = [] - if out_dir.exists(): - images = [f.name for f in out_dir.glob("annotated_*") if f.is_file()] - - return {"status": "ok", "images": sorted(images)} - -@app.get("/api/images/{filename}") -def serve_image(filename: str): - file_path = Path("runs/infer") / filename - if not file_path.exists(): - raise HTTPException(status_code=404, detail="Image not found") - return FileResponse(file_path) - -@app.get("/api/grade") -def grade_predictions(): - # Runs the autograder.py script - predictions_file = Path("runs/infer/predictions.json") - if not predictions_file.exists(): - # Let's run a fallback report if no predictions exist - return { - "total_predictions": 0, - "total_images": 0, - "quality_score": 0.0, - "issues": [ - { - "severity": "WARNING", - "type": "Missing predictions", - "description": "Please run the Auto-Labeler Inference step first to generate prediction files.", - "recommendation": "Go back to Step 4 and run Auto-Labeler." - } - ] - } - - cmd = [ - sys.executable, "autograder.py", - "--predictions", str(predictions_file) - ] - - try: - subprocess.run(cmd, capture_output=True, text=True, check=True) - except subprocess.CalledProcessError as e: - raise HTTPException(status_code=500, detail=f"Autograder failed: {e.stderr}") - - # Read generated quality report - report_file = Path("runs/infer/quality_report.json") - if report_file.exists(): - with open(report_file, "r") as f: - report_data = json.load(f) - - # Map standard report issues to list - issues = [] - for issue_type, count in report_data.get("issue_summary", {}).items(): - if count > 0: - severity = "WARNING" if issue_type in ["low_confidence", "overlapping_different_class"] else "INFO" - desc = f"Found {count} occurrences of {issue_type.replace('_', ' ')}." - rec = "Retrain model with more samples or tune confidence threshold." - if issue_type == "small_detection": - rec = "Verify that extremely small bounding boxes are correct." - elif issue_type == "class_imbalance": - rec = "Balance class frequency in dataset to prevent bias." - - issues.append({ - "severity": severity, - "type": issue_type.upper().replace("_", " "), - "description": desc, - "recommendation": rec - }) - - # Append recommendations - for rec in report_data.get("recommendations", []): - issues.append({ - "severity": "INFO", - "type": "RECOMMENDATION", - "description": rec, - "recommendation": "Review annotation guidelines." - }) - - return { - "total_predictions": report_data.get("total_predictions", 0), - "total_images": report_data.get("total_images", 0), - "quality_score": report_data.get("quality_score", 0.0), - "issues": issues - } - - return {"status": "error", "message": "Failed to read grader output"} - -@app.post("/api/export") -def export_weights(params: ExportParams): - config = load_config() - if config.pipeline_mode == "3d": - from sensorflow.launch_gate_evaluator import LaunchGateEvaluator - evaluator = LaunchGateEvaluator(Path(config.gate_thresholds_path)) - if not evaluator.is_export_allowed(config.sequence_id): - raise HTTPException( - status_code=403, - detail="Export blocked: launch gate not passed. Run quality gate and launch gate first.", - ) - - cmd = [ - sys.executable, "-c", - f"from ultralytics import YOLO; YOLO('{params.weights}').export(format='{params.format}')" - ] - - try: - subprocess.run(cmd, capture_output=True, text=True, check=True) - except subprocess.CalledProcessError as e: - raise HTTPException(status_code=500, detail=f"Export command failed: {e.stderr}") - - base_path = Path(params.weights).parent - exported_file = base_path / f"best.{params.format}" - return {"status": "ok", "exported_file": str(exported_file)} - -class CopilotContextRequest(BaseModel): - context_type: Optional[str] = None - annotation_id: Optional[str] = None - event_id: Optional[str] = None - model_version: Optional[str] = None - extra: dict = {} - -@app.post("/api/copilot/explain") -def copilot_explain(req: Optional[CopilotContextRequest] = None): - # Evidence-based path: evaluation-platform contexts (FP/FN/anomaly/regression/ - # disagreement) route to the labeleval copilot, which handles Ollama being - # unreachable with a deterministic offline analysis. - if req is not None and (req.context_type or req.annotation_id or req.event_id): - from sensorflow.evaluation import copilot as eval_copilot - from sensorflow.evaluation.records import get_store - return eval_copilot.explain(get_store(), req.model_dump()) - - # Legacy path: YOLO studio quality audit. - # 1. Gather configuration details - config = load_config() - - # 2. Gather training details - global TRAINING_LOSSES, TOTAL_EPOCHS - training_summary = { - "epochs_limit": TOTAL_EPOCHS, - "losses_trend": TRAINING_LOSSES, - "logs_snippet": TRAINING_LOGS_ACCUMULATED[-1000:] if TRAINING_LOGS_ACCUMULATED else "No logs recorded." - } - - # 3. Gather grader diagnostics - report_file = Path("runs/infer/quality_report.json") - grader_summary = {} - if report_file.exists(): - try: - with open(report_file, "r") as f: - grader_summary = json.load(f) - except Exception: - pass - - # 4. Construct prompt - prompt = f"""You are an AI Triage Assistant in an autonomous driving computer vision pipeline. -Analyze the following process run context, source dataset settings, and triage/autograding metrics to provide a concise and highly-actionable quality audit report. - -### CONTEXT & CONFIGURATION -- Dataset Config YAML: {config.yaml_path} -- Source Images Path: {config.source_path} - -### TRAINING PROCESS SUMMARY -- Configured Epochs: {training_summary['epochs_limit']} -- Training Loss Trend: {training_summary['losses_trend']} -- Last training log snippet: -``` -{training_summary['logs_snippet']} -``` - -### TRIAGE / AUTO-GRADER RESULTS -{json.dumps(grader_summary, indent=2) if grader_summary else "No auto-grader results yet. (Please run Auto-Labeler and Quality Diagnostics stages first)."} - -Provide a structured, beautifully formatted report in markdown containing: -1. 🔍 **Pipeline & Process Verification**: Brief audit of the training process, if loss decreased properly, etc. -2. 📂 **Source Data Assessment**: Verify if paths and configs match properly. -3. 🛠️ **Triage & Quality Insights**: Summary of major issues (low confidence, small boxes, overlapping labels) and their severity. -4. 📈 **Actionable Recommendations**: Clear suggestions for improvements. - -Keep the response concise, professional, and directly addressable. Do not include introductory conversational fluff.""" - - # 5. Connect to Ollama - endpoints = [ - {"url": "http://dgx-spark.tail16d8d9.ts.net:11434/api/chat", "model": "gemma4:26b"}, - {"url": "http://localhost:11434/api/chat", "model": "gemma4:latest"} - ] - - response_text = "" - error_msg = "" - for ep in endpoints: - try: - payload = { - "model": ep["model"], - "messages": [ - {"role": "user", "content": prompt} - ], - "stream": False - } - res = httpx.post(ep["url"], json=payload, timeout=25.0) - if res.status_code == 200: - response_text = res.json().get("message", {}).get("content", "") - if response_text: - return {"status": "ok", "provider": ep["url"], "analysis": response_text} - except Exception as e: - error_msg += f"[{ep['url']}]: {str(e)}; " - - if not response_text: - # Graceful offline fallback instead of a hard failure. - return { - "status": "ok", - "provider": "offline_deterministic", - "analysis": ( - "## Offline Pipeline Audit (Ollama unreachable)\n\n" - f"- Configured epochs: {training_summary['epochs_limit']}\n" - f"- Loss trend: {training_summary['losses_trend'] or 'no training recorded'}\n" - f"- Grader report: {'present' if grader_summary else 'not generated yet'}\n\n" - "Local LLM endpoints could not be reached " - f"({error_msg.strip()[:300]}). This deterministic summary was " - "generated from the actual run state instead." - ), - } - -MITL_FILE = Path("runs/mitl_annotations.json") - -NVIDIA_SAMPLES = { - "physical_ai": { - "dataset": "nvidia/PhysicalAI-Autonomous-Vehicles", - "frame_index": 4209, - "views": { - "front": "https://images.unsplash.com/photo-1506015391300-4802dc74de2e?w=800&auto=format&fit=crop", - "left": "https://images.unsplash.com/photo-1519074002996-a69e7ac46a42?w=800&auto=format&fit=crop", - "right": "https://images.unsplash.com/photo-1549317661-bd32c8ce0db2?w=800&auto=format&fit=crop" - }, - "telemetry": { - "lat": 37.774929, - "lon": -122.419416, - "speed_kmh": 35.8, - "accel_mps2": 1.25, - "imu_pitch": 0.02, - "imu_roll": -0.01, - "trajectory": [ - {"x": 0.0, "y": 0.0}, - {"x": 1.1, "y": 5.2}, - {"x": 2.2, "y": 10.4}, - {"x": 3.4, "y": 15.6}, - {"x": 4.5, "y": 20.8} - ] - }, - "coc_trace": "The vehicle is approaching a signalized intersection. Traffic lights are green, but a pedestrian is detected near the curb at coordinates [4.5, 20.8] moving towards the crosswalk. Recommendation: Decelerate from 35 km/h to 20 km/h to prepare for potential crossing, maintaining safe headway.", - "annotations": [ - {"id": 1, "label": "pedestrian", "box": [120, 240, 60, 120], "conf": 0.89}, - {"id": 2, "label": "car", "box": [340, 220, 150, 100], "conf": 0.95}, - {"id": 3, "label": "traffic_light", "box": [500, 80, 30, 60], "conf": 0.92} - ] - }, - "nurec": { - "dataset": "nvidia/PhysicalAI-Autonomous-Vehicles-NuRec", - "frame_index": 781, - "views": { - "front": "https://images.unsplash.com/photo-1568605117036-5fe5e7bab0b7?w=800&auto=format&fit=crop", - "left": "https://images.unsplash.com/photo-1552519507-da3b142c6e3d?w=800&auto=format&fit=crop", - "right": "https://images.unsplash.com/photo-1503376780353-7e6692767b70?w=800&auto=format&fit=crop" - }, - "telemetry": { - "lat": 37.783312, - "lon": -122.416733, - "speed_kmh": 48.2, - "accel_mps2": -0.15, - "imu_pitch": -0.01, - "imu_roll": 0.00, - "trajectory": [ - {"x": 0.0, "y": 0.0}, - {"x": 0.2, "y": 7.0}, - {"x": 0.4, "y": 14.1}, - {"x": 0.5, "y": 21.2}, - {"x": 0.6, "y": 28.3} - ] - }, - "coc_trace": "Reconstructed scene matches ground truth trajectory with <2% positional drift. Dynamic vehicle models are rendered as rigid boxes. Pedestrian mesh shows minor deformation anomalies at distance >15m. Recommendation: Re-render scene with updated neural radiance field weights to reduce floaters.", - "annotations": [ - {"id": 1, "label": "car", "box": [300, 180, 180, 140], "conf": 0.94}, - {"id": 2, "label": "truck", "box": [50, 150, 220, 180], "conf": 0.81} - ] - } -} - -class SaveMitlRequest(BaseModel): - dataset_type: str - annotations: List[dict] - coc_trace: str - -class EvaluateMitlRequest(BaseModel): - annotations: List[dict] - coc_trace: str - original_coc: str - -@app.post("/api/nvidia/load") -def load_nvidia_dataset(params: dict): - dataset = params.get("dataset", "physical_ai") - if dataset not in NVIDIA_SAMPLES: - raise HTTPException(status_code=400, detail="Invalid NVIDIA dataset name") - - try: - from huggingface_hub import HfApi - api = HfApi() - api.space_info("nvidia/Alpamayo-R1-10B") - except Exception: - pass - - sample = NVIDIA_SAMPLES[dataset] - with open(MITL_FILE, "w") as f: - json.dump(sample, f, indent=2) - - return {"status": "ok", "message": f"Successfully ingested {sample['dataset']} samples from HF.", "data": sample} - -@app.get("/api/mitl/annotations") -def get_mitl_annotations(): - if MITL_FILE.exists(): - try: - with open(MITL_FILE, "r") as f: - return json.load(f) - except Exception: - pass - return NVIDIA_SAMPLES["physical_ai"] - -@app.post("/api/mitl/annotations") -def save_mitl_annotations(req: SaveMitlRequest): - current_data = {} - if MITL_FILE.exists(): - try: - with open(MITL_FILE, "r") as f: - current_data = json.load(f) - except Exception: - pass - - if not current_data or current_data.get("dataset") != NVIDIA_SAMPLES.get(req.dataset_type, {}).get("dataset"): - current_data = NVIDIA_SAMPLES.get(req.dataset_type, {}).copy() - - current_data["annotations"] = req.annotations - current_data["coc_trace"] = req.coc_trace - - with open(MITL_FILE, "w") as f: - json.dump(current_data, f, indent=2) - - return {"status": "ok", "message": "Man-in-the-Loop annotations saved."} - -@app.post("/api/mitl/evaluate") -def evaluate_mitl(req: EvaluateMitlRequest): - prompt = f"""You are the Lead QA Auditor evaluating a Man-in-the-Loop (MITL) annotation session in an Autonomous Vehicle perception team. -Compare the original and human-modified reasoning traces/annotations and provide a concise critique on: -1. Whether the human's changes to the Chain-of-Causation (CoC) reasoning trace make logical sense based on the visual labels. -2. Any warnings or potential issues with the edited labels or the updated reasoning. - -ORIGINAL COC REASONING: -{req.original_coc} - -UPDATED COC REASONING: -{req.coc_trace} - -UPDATED ANNOTATIONS LIST: -{json.dumps(req.annotations, indent=2)} - -Provide a professional, clear critique in markdown. Be concise and focus purely on verification and recommendations.""" - - endpoints = [ - {"url": "http://dgx-spark.tail16d8d9.ts.net:11434/api/chat", "model": "gemma4:26b"}, - {"url": "http://localhost:11434/api/chat", "model": "gemma4:latest"} - ] - - response_text = "" - error_msg = "" - for ep in endpoints: - try: - payload = { - "model": ep["model"], - "messages": [ - {"role": "user", "content": prompt} - ], - "stream": False - } - res = httpx.post(ep["url"], json=payload, timeout=25.0) - if res.status_code == 200: - response_text = res.json().get("message", {}).get("content", "") - if response_text: - return {"status": "ok", "provider": ep["url"], "critique": response_text} - except Exception as e: - error_msg += f"[{ep['url']}]: {str(e)}; " - - if not response_text: - return { - "status": "warning", - "provider": "Mock Triage Auditor", - "critique": f"### ⚠️ Local LLM Evaluation Offline\\nFailed to connect to Gemma 4 via Ollama. Critique preview: Human adjustments to the Chain-of-Causation reasoning trace were saved. Telemetry coordinates align with the {len(req.annotations)} updated label assets. No immediate class conflicts detected." - } - -@app.get("/api/benchmark/compare") -def benchmark_compare(dataset: str = "physical_ai"): - config = load_config() - metric_path = Path("runs/pipeline") / config.sequence_id / "benchmark" / "metric_card.json" - if metric_path.exists(): - with open(metric_path) as f: - metrics = json.load(f) - return { - "status": "ok", - "dataset": dataset, - "source": "live_benchmark", - "benchmarks": { - "sensorflow_3d_pipeline": { - "name": "Sensorflow 3D Pipeline", - "type": "SAM + LiDAR + Tracker", - "map_3d": metrics.get("map_3d", 0), - "mar_3d": metrics.get("mar_3d", 0), - "mean_iou_3d": metrics.get("mean_iou_3d", 0), - "orientation_error_deg": metrics.get("orientation_error_deg", 0), - "position_error_m": metrics.get("position_error_m", 0), - "id_swap_rate": metrics.get("id_swap_rate", 0), - "track_fragmentation_rate": metrics.get("track_fragmentation_rate", 0), - "process_units": metrics.get("process_units", 0), - "compute_cycles": metrics.get("compute_cycles", 0), - } - }, - "metrics": metrics, - } - - comparison_data = { - "yolov8m.pt": { - "name": "YOLOv8 Medium", - "type": "Standard Object Detector", - "latency_ms": 12.4, - "map50": 0.76, - "risk_weighted_recall": 0.71, - "recall_critical_distance": 0.68, - "coc_support": "No", - "vru_recall": 0.74 - }, - "nvidia/Alpamayo-1.5-10B": { - "name": "Alpamayo-1.5-10B", - "type": "AV Trajectory & Labeller", - "latency_ms": 115.2, - "map50": 0.89, - "risk_weighted_recall": 0.91, - "recall_critical_distance": 0.94, - "coc_support": "No", - "vru_recall": 0.92 - }, - "nvidia/Alpamayo-R1-10B": { - "name": "Alpamayo-R1-10B (VLM)", - "type": "Chain-of-Causation VLM", - "latency_ms": 450.8, - "map50": 0.94, - "risk_weighted_recall": 0.97, - "recall_critical_distance": 0.98, - "coc_support": "Yes (700k traces)", - "vru_recall": 0.98 - } - } - return {"status": "ok", "dataset": dataset, "source": "fallback_mock", "benchmarks": comparison_data} - -class SavePipelineToolsRequest(BaseModel): - annotation_tool: str - training_framework: str - validation_method: str - -DATASET_METADATA_STORE = { - "local": { - "dataset_type": "local", - "name": "Local Directory (Default YOLO COCO8)", - "total_rows_source": 128, - "loaded_rows": 128, - "ingestion_pct": "100.0%", - "total_annotations": 432, - "classes": ["person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck"], - "class_counts": {"car": 184, "person": 112, "truck": 58, "bicycle": 42, "motorcycle": 24, "bus": 12}, - "sensor_modality": "Monocular Front RGB Camera (640x640)", - "storage_footprint": "6.5 MB", - "format": "YOLOv8 TXT / COCO YAML", - "licensing": "Ultralytics AGPL-3.0 / Open Source", - "geographic_coverage": "Local Drive (/DrivingRepo/data)", - "weather_conditions": "Daytime / Mixed", - "annotation_tool": "Auto-Labeler + Manual Bounding Box", - "description": "Standard lightweight object detection dataset for rapid local testing and mobile/edge training." - }, - "waymo": { - "dataset_type": "waymo", - "name": "Waymo Open Dataset (v1.4)", - "total_rows_source": 23410, - "loaded_rows": 1500, - "ingestion_pct": "6.4%", - "total_annotations": 2840000, - "classes": ["vehicle", "pedestrian", "cyclist", "sign"], - "class_counts": {"vehicle": 1620000, "pedestrian": 890000, "cyclist": 210000, "sign": 120000}, - "sensor_modality": "5x LiDARs (Top & 4 Sides) + 5x High-Res Pin-hole Cameras + IMU/GNSS Telemetry", - "storage_footprint": "1.2 TB", - "format": "TFRecord / Protocol Buffers", - "licensing": "Waymo Non-Commercial Research License", - "geographic_coverage": "San Francisco, Phoenix, Mountain View (CA/AZ)", - "weather_conditions": "Day, Night, Rain, Fog, Overcast", - "annotation_tool": "Auto-Labeled 3D LiDAR & 2D Bounding Boxes", - "description": "High-resolution multi-sensor dataset capturing diverse urban driving environments across California and Arizona." - }, - "alpamayo": { - "dataset_type": "alpamayo", - "name": "NVIDIA Alpamayo Physical AI AV Dataset", - "total_rows_source": 700000, - "loaded_rows": 4209, - "ingestion_pct": "0.6%", - "total_annotations": 12500000, - "classes": ["car", "truck", "pedestrian", "cyclist", "traffic_light", "traffic_sign"], - "class_counts": {"car": 6800000, "pedestrian": 2900000, "traffic_light": 1400000, "truck": 850000, "cyclist": 550000}, - "sensor_modality": "8x Surround Cameras (4K 60fps) + 3D LiDAR + CAN-Bus Telemetry + Chain-of-Causation Reasoning Traces", - "storage_footprint": "450 GB", - "format": "Parquet + HDF5 + JSON CoC Traces", - "licensing": "NVIDIA Physical AI Open License", - "geographic_coverage": "SF Bay Area, Highway 101, Sunnyvale Intersections", - "weather_conditions": "Clear, Rain, Dusk, Night, Construction Zones", - "annotation_tool": "NeMo Studio Reasoning Engine + CoC Automated Traces", - "description": "Physical AI reasoning dataset pairing multi-camera sensor video with Chain-of-Causation natural language decision traces." - }, - "a2d2": { - "dataset_type": "a2d2", - "name": "Audi A2D2 Autonomous Driving Dataset", - "total_rows_source": 41200, - "loaded_rows": 2500, - "ingestion_pct": "6.1%", - "total_annotations": 380000, - "classes": ["car", "pedestrian", "pedestrian_group", "truck", "bus", "bicycle", "motorcycle", "traffic_sign", "signal", "obstacle"], - "class_counts": {"car": 210000, "traffic_sign": 75000, "pedestrian": 45000, "truck": 28000, "signal": 22000}, - "sensor_modality": "6x Cameras (1920x1208 @ 30fps) + 5x LiDAR Sensors + Vehicle Bus Telemetry (Steering, Speed, Yaw)", - "storage_footprint": "2.3 TB", - "format": "HDF5 + PNG + JSON", - "licensing": "Audi A2D2 Non-Commercial License", - "geographic_coverage": "Ingolstadt, Munich, Gaimersheim (Germany)", - "weather_conditions": "European Highway, Suburban, Urban Snow & Rain", - "annotation_tool": "3D Bounding Boxes & Semantic Segmentation Maps", - "description": "European autonomous driving dataset featuring synchronized 3D LiDAR, 2D camera images, and vehicle bus sensor data." - }, - "ssam": { - "dataset_type": "ssam", - "name": "California Statewide SSAM Intersection Safety Dataset", - "total_rows_source": 845, - "loaded_rows": 845, - "ingestion_pct": "100.0%", - "total_annotations": 4225, - "classes": ["rear_end", "angle", "sideswipe", "head_on", "pedestrian_cross"], - "class_counts": {"rear_end": 1840, "angle": 1220, "sideswipe": 680, "head_on": 320, "pedestrian_cross": 165}, - "sensor_modality": "SSAM Traffic Conflict Telemetry & Intersection Video Log Records", - "storage_footprint": "18.4 MB", - "format": "GeoJSON + SSAM XML/CSV", - "licensing": "Federal Highway Administration (FHWA) Public Domain", - "geographic_coverage": "Statewide California Intersection Corridors", - "weather_conditions": "Multi-year Statewide Microclimates", - "annotation_tool": "Surrogate Safety Assessment Model (SSAM) Analytics Engine", - "description": "Statewide collision conflict dataset tracking Time-to-Collision (TTC) and Post-Encroachment Time (PET) across intersections." - } -} - -@app.get("/api/dataset/details") -def get_dataset_details(type: str = "local"): - metadata = DATASET_METADATA_STORE.get(type, DATASET_METADATA_STORE["local"]) - return {"status": "ok", "metadata": metadata} - -@app.post("/api/dataset/preprocess") -def preprocess_dataset(params: dict): - dataset_type = params.get("dataset_type", "local") - import time - time.sleep(0.3) - - meta = DATASET_METADATA_STORE.get(dataset_type, DATASET_METADATA_STORE["local"]) - - return { - "status": "ok", - "message": f"Dataset {meta['name']} loaded and preprocessed successfully.", - "dataset_type": dataset_type, - "total_frames": meta["loaded_rows"], - "metadata": meta - } - -@app.post("/api/dataset/save-pipeline-tools") -def save_pipeline_tools(req: SavePipelineToolsRequest): - # Save the selected tools back to a session or local configuration - # In a real environment, we'd update CONFIG_PATH or StudioConfig instance - config = load_config() - # We can write it to the config file by updating it - config_dict = config.dict() - config_dict["annotation_tool"] = req.annotation_tool - config_dict["training_framework"] = req.training_framework - config_dict["validation_method"] = req.validation_method - - with open(CONFIG_PATH, "w") as f: - json.dump(config_dict, f, indent=2) - - return {"status": "ok", "message": "Pipeline tools and methodologies saved successfully."} - -MCP_CONFIG_PATH = Path(os.path.expanduser("~/.gemini/config/mcp_config.json")) - -class SaveMcpRequest(BaseModel): - config_json: str - -class ToggleMcpRequest(BaseModel): - server_name: str - active: bool - -@app.get("/api/mcp/config") -def get_mcp_config(): - if not MCP_CONFIG_PATH.exists(): - fallback_data = { - "mcpServers": { - "ollama-local": { - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-ollama", "--host", "localhost"] - }, - "ollama-spark-disabled": { - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-ollama", "--host", "dgx-spark.tail16d8d9.ts.net"] - } - } - } - return {"status": "ok", "raw": json.dumps(fallback_data, indent=2), "file_exists": False, "config": fallback_data} - - try: - with open(MCP_CONFIG_PATH, "r") as f: - content = f.read() - try: - import re - clean_content = re.sub(r'//.*', '', content) - clean_content = re.sub(r'/\*.*?\*/', '', clean_content, flags=re.DOTALL) - if not clean_content.strip(): - data = {"mcpServers": {}} - else: - data = json.loads(clean_content) - except Exception: - data = {"mcpServers": {}} - return {"status": "ok", "raw": content, "file_exists": True, "config": data} - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to read MCP config: {str(e)}") - -@app.post("/api/mcp/save") -def save_mcp_config(req: SaveMcpRequest): - try: - data = json.loads(req.config_json) - MCP_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True) - with open(MCP_CONFIG_PATH, "w") as f: - f.write(req.config_json) - return {"status": "ok", "message": "MCP config successfully saved."} - except json.JSONDecodeError as e: - raise HTTPException(status_code=400, detail=f"Invalid JSON content: {str(e)}") - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to save MCP config: {str(e)}") - -@app.post("/api/mcp/toggle") -def toggle_mcp_server(req: ToggleMcpRequest): - if not MCP_CONFIG_PATH.exists(): - return {"status": "warning", "message": "MCP config file does not exist, changes simulated."} - - try: - with open(MCP_CONFIG_PATH, "r") as f: - data = json.load(f) - - servers = data.get("mcpServers", {}) - target = req.server_name - - if req.active: - disabled_key = f"{target}-disabled" - if disabled_key in servers: - servers[target] = servers.pop(disabled_key) - elif target not in servers: - servers[target] = {"command": "node", "args": []} - else: - if target in servers: - disabled_key = f"{target}-disabled" - servers[disabled_key] = servers.pop(target) - - data["mcpServers"] = servers - - with open(MCP_CONFIG_PATH, "w") as f: - json.dump(data, f, indent=2) - - return {"status": "ok", "message": f"Server {target} successfully toggled.", "config": data} - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to toggle MCP server: {str(e)}") - -SSAM_STREETS_PATH = Path("ssam_streets.json") - -class AnnotateStreetRequest(BaseModel): - street_name: str - manual_annotation: str - -# --------------------------------------------------------------------------- -# Statewide California SSAM dataset – representative intersections across -# all 58 counties with realistic lat/lng, conflict metrics, and severity. -# --------------------------------------------------------------------------- -import random as _rng -_rng.seed(42) - -_CA_INTERSECTIONS = [ - # Northern California - {"street_name": "Market St & 4th St", "county": "San Francisco", "lat": 37.7855, "lng": -122.4057, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.2, "max_speed": 12.5}, - {"street_name": "Mission St & 16th St", "county": "San Francisco", "lat": 37.7651, "lng": -122.4197, "conflict_type": "Lane-change", "min_ttc": 0.5, "min_pet": 0.9, "max_speed": 15.0}, - {"street_name": "Van Ness Ave & Geary Blvd", "county": "San Francisco", "lat": 37.7863, "lng": -122.4213, "conflict_type": "Crossing", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 11.0}, - {"street_name": "Broadway & Embarcadero", "county": "San Francisco", "lat": 37.7987, "lng": -122.3974, "conflict_type": "Rear-end", "min_ttc": 1.4, "min_pet": 3.2, "max_speed": 9.0}, - {"street_name": "Telegraph Ave & Ashby Ave", "county": "Alameda", "lat": 37.8559, "lng": -122.2600, "conflict_type": "Rear-end", "min_ttc": 1.6, "min_pet": 4.5, "max_speed": 8.0}, - {"street_name": "International Blvd & 73rd Ave", "county": "Alameda", "lat": 37.7530, "lng": -122.1735, "conflict_type": "Crossing", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.2}, - {"street_name": "Hegenberger Rd & 98th Ave", "county": "Alameda", "lat": 37.7378, "lng": -122.1848, "conflict_type": "Lane-change", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 13.1}, - {"street_name": "El Camino Real & University Ave", "county": "Santa Clara", "lat": 37.4445, "lng": -122.1633, "conflict_type": "Crossing", "min_ttc": 1.2, "min_pet": 2.3, "max_speed": 10.5}, - {"street_name": "Stevens Creek Blvd & De Anza Blvd", "county": "Santa Clara", "lat": 37.3232, "lng": -122.0316, "conflict_type": "Rear-end", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.8}, - {"street_name": "Capitol Expwy & Story Rd", "county": "Santa Clara", "lat": 37.3283, "lng": -121.8500, "conflict_type": "Crossing", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 13.5}, - {"street_name": "Bascom Ave & Hamilton Ave", "county": "Santa Clara", "lat": 37.3022, "lng": -121.9319, "conflict_type": "Lane-change", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.8}, - {"street_name": "Broadway Blvd & El Camino", "county": "San Mateo", "lat": 37.4865, "lng": -122.2319, "conflict_type": "Rear-end", "min_ttc": 1.6, "min_pet": 4.5, "max_speed": 8.0}, - {"street_name": "Hillsdale Blvd & El Camino", "county": "San Mateo", "lat": 37.5395, "lng": -122.2944, "conflict_type": "Crossing", "min_ttc": 1.1, "min_pet": 2.1, "max_speed": 10.2}, - {"street_name": "Sir Francis Drake Blvd & Hwy 101", "county": "Marin", "lat": 38.0028, "lng": -122.5236, "conflict_type": "Lane-change", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 14.0}, - {"street_name": "Lincoln Ave & 3rd St", "county": "Marin", "lat": 37.9253, "lng": -122.5113, "conflict_type": "Rear-end", "min_ttc": 1.5, "min_pet": 3.8, "max_speed": 7.5}, - {"street_name": "Sonoma Blvd & Tennessee St", "county": "Solano", "lat": 38.0993, "lng": -122.2567, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.6, "max_speed": 12.0}, - {"street_name": "Texas St & Springs Rd", "county": "Solano", "lat": 38.3505, "lng": -121.9619, "conflict_type": "Rear-end", "min_ttc": 1.4, "min_pet": 3.5, "max_speed": 8.5}, - {"street_name": "Petaluma Blvd S & D St", "county": "Sonoma", "lat": 38.2358, "lng": -122.6404, "conflict_type": "Crossing", "min_ttc": 1.2, "min_pet": 2.4, "max_speed": 9.5}, - {"street_name": "Mendocino Ave & College Ave", "county": "Sonoma", "lat": 38.4545, "lng": -122.7132, "conflict_type": "Lane-change", "min_ttc": 1.0, "min_pet": 1.9, "max_speed": 11.0}, - {"street_name": "Contra Costa Blvd & Treat Blvd", "county": "Contra Costa", "lat": 37.9580, "lng": -122.0569, "conflict_type": "Crossing", "min_ttc": 0.7, "min_pet": 1.2, "max_speed": 13.8}, - {"street_name": "San Pablo Ave & Cutting Blvd", "county": "Contra Costa", "lat": 37.9302, "lng": -122.3564, "conflict_type": "Rear-end", "min_ttc": 1.3, "min_pet": 3.0, "max_speed": 9.2}, - {"street_name": "Napa Valley Hwy & Imola Ave", "county": "Napa", "lat": 38.2877, "lng": -122.2757, "conflict_type": "Crossing", "min_ttc": 1.5, "min_pet": 3.6, "max_speed": 8.0}, - # Sacramento / Central Valley - {"street_name": "Stockton Blvd & Fruitridge Rd", "county": "Sacramento", "lat": 38.5246, "lng": -121.4579, "conflict_type": "Crossing", "min_ttc": 0.6, "min_pet": 0.9, "max_speed": 15.2}, - {"street_name": "Watt Ave & Arden Way", "county": "Sacramento", "lat": 38.5987, "lng": -121.3832, "conflict_type": "Rear-end", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 13.0}, - {"street_name": "Florin Rd & Power Inn Rd", "county": "Sacramento", "lat": 38.4953, "lng": -121.4200, "conflict_type": "Lane-change", "min_ttc": 1.1, "min_pet": 2.2, "max_speed": 11.5}, - {"street_name": "J St & 16th St", "county": "Sacramento", "lat": 38.5767, "lng": -121.4859, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 10.0}, - {"street_name": "McHenry Ave & Briggsmore Ave", "county": "Stanislaus", "lat": 37.6649, "lng": -120.9971, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 12.8}, - {"street_name": "Yosemite Blvd & Coffee Rd", "county": "Stanislaus", "lat": 37.6182, "lng": -120.9517, "conflict_type": "Rear-end", "min_ttc": 1.2, "min_pet": 2.5, "max_speed": 10.0}, - {"street_name": "Pacific Ave & March Ln", "county": "San Joaquin", "lat": 37.9914, "lng": -121.3258, "conflict_type": "Crossing", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 14.0}, - {"street_name": "Hammer Ln & I-5", "county": "San Joaquin", "lat": 38.0204, "lng": -121.3536, "conflict_type": "Lane-change", "min_ttc": 0.5, "min_pet": 0.8, "max_speed": 16.5}, - {"street_name": "Herndon Ave & Blackstone Ave", "county": "Fresno", "lat": 36.8267, "lng": -119.7886, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.4, "max_speed": 12.5}, - {"street_name": "Shaw Ave & Cedar Ave", "county": "Fresno", "lat": 36.8078, "lng": -119.7696, "conflict_type": "Rear-end", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.2}, - {"street_name": "Kings Canyon Rd & Clovis Ave", "county": "Fresno", "lat": 36.7268, "lng": -119.7020, "conflict_type": "Lane-change", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.8}, - {"street_name": "Olive Ave & Chester Ave", "county": "Kern", "lat": 35.3863, "lng": -119.0188, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.0}, - {"street_name": "White Ln & Wible Rd", "county": "Kern", "lat": 35.3256, "lng": -119.0476, "conflict_type": "Rear-end", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 10.8}, - {"street_name": "Ming Ave & Ashe Rd", "county": "Kern", "lat": 35.3417, "lng": -119.0713, "conflict_type": "Lane-change", "min_ttc": 0.7, "min_pet": 1.2, "max_speed": 13.5}, - {"street_name": "Mooney Blvd & Caldwell Ave", "county": "Tulare", "lat": 36.3269, "lng": -119.2883, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.7, "max_speed": 11.5}, - {"street_name": "G St & Main St", "county": "Merced", "lat": 37.3022, "lng": -120.4829, "conflict_type": "Rear-end", "min_ttc": 1.3, "min_pet": 2.9, "max_speed": 9.5}, - {"street_name": "Main St & Yosemite Ave", "county": "Madera", "lat": 36.9627, "lng": -120.0608, "conflict_type": "Crossing", "min_ttc": 1.4, "min_pet": 3.1, "max_speed": 8.8}, - {"street_name": "Court St & D St", "county": "Kings", "lat": 36.3292, "lng": -119.6431, "conflict_type": "Lane-change", "min_ttc": 1.2, "min_pet": 2.4, "max_speed": 10.2}, - # Southern California - {"street_name": "Hollywood Blvd & Highland Ave", "county": "Los Angeles", "lat": 34.1017, "lng": -118.3389, "conflict_type": "Crossing", "min_ttc": 0.4, "min_pet": 0.7, "max_speed": 16.0}, - {"street_name": "Sunset Blvd & Vine St", "county": "Los Angeles", "lat": 34.0982, "lng": -118.3266, "conflict_type": "Lane-change", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.5}, - {"street_name": "La Brea Ave & Wilshire Blvd", "county": "Los Angeles", "lat": 34.0621, "lng": -118.3441, "conflict_type": "Rear-end", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 13.0}, - {"street_name": "Figueroa St & 7th St", "county": "Los Angeles", "lat": 34.0490, "lng": -118.2590, "conflict_type": "Crossing", "min_ttc": 0.5, "min_pet": 0.8, "max_speed": 15.5}, - {"street_name": "Vermont Ave & MLK Blvd", "county": "Los Angeles", "lat": 33.9917, "lng": -118.2917, "conflict_type": "Crossing", "min_ttc": 0.3, "min_pet": 0.5, "max_speed": 17.2}, - {"street_name": "Century Blvd & Crenshaw Blvd", "county": "Los Angeles", "lat": 33.9461, "lng": -118.3302, "conflict_type": "Lane-change", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 14.0}, - {"street_name": "PCH & Sunset Blvd", "county": "Los Angeles", "lat": 34.0411, "lng": -118.5266, "conflict_type": "Rear-end", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.0}, - {"street_name": "Sepulveda Blvd & National Blvd", "county": "Los Angeles", "lat": 34.0223, "lng": -118.3973, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.5}, - {"street_name": "Harbor Blvd & Chapman Ave", "county": "Orange", "lat": 33.8316, "lng": -117.9178, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 12.8}, - {"street_name": "Beach Blvd & Edinger Ave", "county": "Orange", "lat": 33.7582, "lng": -117.9974, "conflict_type": "Rear-end", "min_ttc": 1.1, "min_pet": 2.1, "max_speed": 10.5}, - {"street_name": "Bristol St & MacArthur Blvd", "county": "Orange", "lat": 33.6814, "lng": -117.8855, "conflict_type": "Lane-change", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.2}, - {"street_name": "Euclid St & Lincoln Ave", "county": "Orange", "lat": 33.8381, "lng": -117.9462, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.7, "max_speed": 11.5}, - {"street_name": "University Ave & Iowa Ave", "county": "Riverside", "lat": 33.9716, "lng": -117.3431, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.2}, - {"street_name": "Van Buren Blvd & Arlington Ave", "county": "Riverside", "lat": 33.9250, "lng": -117.4347, "conflict_type": "Rear-end", "min_ttc": 1.2, "min_pet": 2.5, "max_speed": 10.0}, - {"street_name": "Date Palm Dr & Ramon Rd", "county": "Riverside", "lat": 33.7381, "lng": -116.3622, "conflict_type": "Lane-change", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.8}, - {"street_name": "Mt Vernon Ave & Mill St", "county": "San Bernardino", "lat": 34.0689, "lng": -117.2838, "conflict_type": "Crossing", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 13.5}, - {"street_name": "Foothill Blvd & Vineyard Ave", "county": "San Bernardino", "lat": 34.1069, "lng": -117.5884, "conflict_type": "Rear-end", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5}, - {"street_name": "Bear Valley Rd & Hesperia Rd", "county": "San Bernardino", "lat": 34.4504, "lng": -117.3120, "conflict_type": "Lane-change", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 14.0}, - {"street_name": "El Cajon Blvd & 30th St", "county": "San Diego", "lat": 32.7594, "lng": -117.1297, "conflict_type": "Crossing", "min_ttc": 0.7, "min_pet": 1.2, "max_speed": 13.0}, - {"street_name": "Balboa Ave & Genesee Ave", "county": "San Diego", "lat": 32.8207, "lng": -117.1810, "conflict_type": "Rear-end", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 10.8}, - {"street_name": "University Ave & Fairmount Ave", "county": "San Diego", "lat": 32.7490, "lng": -117.1062, "conflict_type": "Lane-change", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.5}, - {"street_name": "Broadway & E St", "county": "San Diego", "lat": 32.7170, "lng": -117.1631, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5}, - {"street_name": "Palm Canyon Dr & Tahquitz Canyon Way", "county": "Riverside", "lat": 33.8254, "lng": -116.5453, "conflict_type": "Crossing", "min_ttc": 1.4, "min_pet": 3.2, "max_speed": 8.5}, - {"street_name": "State St & De La Vina St", "county": "Santa Barbara", "lat": 34.4275, "lng": -119.7089, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.7, "max_speed": 9.8}, - {"street_name": "Milpas St & Haley St", "county": "Santa Barbara", "lat": 34.4188, "lng": -119.6879, "conflict_type": "Rear-end", "min_ttc": 1.5, "min_pet": 3.5, "max_speed": 8.2}, - {"street_name": "Ventura Blvd & Laurel Canyon Blvd", "county": "Ventura", "lat": 34.1543, "lng": -118.3975, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 13.0}, - {"street_name": "Victoria Ave & Telephone Rd", "county": "Ventura", "lat": 34.2599, "lng": -119.2187, "conflict_type": "Lane-change", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5}, - {"street_name": "Johnson Ave & Grand Ave", "county": "San Luis Obispo", "lat": 35.2697, "lng": -120.6597, "conflict_type": "Crossing", "min_ttc": 1.4, "min_pet": 3.0, "max_speed": 9.0}, - {"street_name": "El Camino Real & Atascadero Ave", "county": "San Luis Obispo", "lat": 35.4872, "lng": -120.6726, "conflict_type": "Rear-end", "min_ttc": 1.6, "min_pet": 4.0, "max_speed": 7.5}, - {"street_name": "Main St & Pine St", "county": "Monterey", "lat": 36.5949, "lng": -121.8957, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5}, - {"street_name": "N Main St & Alisal St", "county": "Monterey", "lat": 36.6705, "lng": -121.6558, "conflict_type": "Rear-end", "min_ttc": 1.0, "min_pet": 1.7, "max_speed": 11.8}, - {"street_name": "River Park Dr & W Sacramento Ave", "county": "Yolo", "lat": 38.5791, "lng": -121.5520, "conflict_type": "Crossing", "min_ttc": 1.2, "min_pet": 2.3, "max_speed": 10.5}, - {"street_name": "Bridge St & Main St", "county": "Yuba", "lat": 39.1394, "lng": -121.6147, "conflict_type": "Rear-end", "min_ttc": 1.5, "min_pet": 3.6, "max_speed": 8.0}, - {"street_name": "10th St & E St", "county": "Yuba", "lat": 39.1413, "lng": -121.6093, "conflict_type": "Crossing", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 10.8}, - {"street_name": "Colusa Ave & Sutter St", "county": "Sutter", "lat": 39.1451, "lng": -121.6207, "conflict_type": "Lane-change", "min_ttc": 1.3, "min_pet": 2.7, "max_speed": 9.5}, - {"street_name": "East Ave & Lindo Channel", "county": "Butte", "lat": 39.7520, "lng": -121.8089, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5}, - {"street_name": "Mangrove Ave & E 20th St", "county": "Butte", "lat": 39.7593, "lng": -121.8387, "conflict_type": "Rear-end", "min_ttc": 1.2, "min_pet": 2.4, "max_speed": 10.2}, - {"street_name": "Hilltop Dr & Cypress Ave", "county": "Shasta", "lat": 40.5889, "lng": -122.3503, "conflict_type": "Crossing", "min_ttc": 1.1, "min_pet": 2.1, "max_speed": 10.8}, - {"street_name": "Market St & Placer St", "county": "Shasta", "lat": 40.5855, "lng": -122.3916, "conflict_type": "Lane-change", "min_ttc": 1.4, "min_pet": 3.2, "max_speed": 8.5}, - {"street_name": "Pine St & S Oregon St", "county": "Siskiyou", "lat": 41.7271, "lng": -122.6362, "conflict_type": "Rear-end", "min_ttc": 1.6, "min_pet": 4.2, "max_speed": 7.0}, - {"street_name": "S Mt Shasta Blvd & E Alma St", "county": "Siskiyou", "lat": 41.3132, "lng": -122.3103, "conflict_type": "Crossing", "min_ttc": 1.5, "min_pet": 3.5, "max_speed": 8.0}, - {"street_name": "Riverside Dr & Court St", "county": "Humboldt", "lat": 40.7854, "lng": -124.1576, "conflict_type": "Crossing", "min_ttc": 1.2, "min_pet": 2.3, "max_speed": 10.5}, - {"street_name": "4th St & C St", "county": "Humboldt", "lat": 40.7872, "lng": -124.1565, "conflict_type": "Rear-end", "min_ttc": 1.4, "min_pet": 3.0, "max_speed": 9.0}, - {"street_name": "Main St & Perkins St", "county": "Mendocino", "lat": 39.1503, "lng": -123.2072, "conflict_type": "Crossing", "min_ttc": 1.5, "min_pet": 3.5, "max_speed": 8.2}, - {"street_name": "S State St & Perkins St", "county": "Mendocino", "lat": 39.1481, "lng": -123.2052, "conflict_type": "Lane-change", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5}, - {"street_name": "Spring St & Washington St", "county": "Nevada", "lat": 39.2627, "lng": -121.0172, "conflict_type": "Crossing", "min_ttc": 1.4, "min_pet": 3.1, "max_speed": 8.8}, - {"street_name": "Auburn Folsom Rd & Douglas Blvd", "county": "Placer", "lat": 38.7516, "lng": -121.2563, "conflict_type": "Rear-end", "min_ttc": 1.1, "min_pet": 2.1, "max_speed": 11.0}, - {"street_name": "Lincoln Way & Sunrise Ave", "county": "Placer", "lat": 38.7360, "lng": -121.2301, "conflict_type": "Lane-change", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.5}, - {"street_name": "Placerville Dr & Missouri Flat Rd", "county": "El Dorado", "lat": 38.7222, "lng": -120.8263, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.7, "max_speed": 9.8}, - {"street_name": "Main St & Jackson Gate Rd", "county": "Amador", "lat": 38.3487, "lng": -120.7732, "conflict_type": "Rear-end", "min_ttc": 1.6, "min_pet": 4.0, "max_speed": 7.5}, - {"street_name": "Main St & Mono Way", "county": "Tuolumne", "lat": 37.9817, "lng": -120.3825, "conflict_type": "Crossing", "min_ttc": 1.4, "min_pet": 3.2, "max_speed": 8.5}, - {"street_name": "E St & 12th St", "county": "Imperial", "lat": 32.7924, "lng": -115.5631, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5}, - {"street_name": "Main St & 8th St", "county": "Imperial", "lat": 32.8426, "lng": -115.5681, "conflict_type": "Rear-end", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5}, - {"street_name": "Barstow Rd & Main St", "county": "San Bernardino", "lat": 34.8987, "lng": -117.0226, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.5}, - # Additional urban hotspots - {"street_name": "Rosecrans Ave & Hawthorne Blvd", "county": "Los Angeles", "lat": 33.9014, "lng": -118.3525, "conflict_type": "Crossing", "min_ttc": 0.5, "min_pet": 0.8, "max_speed": 15.5}, - {"street_name": "Atlantic Ave & Florence Ave", "county": "Los Angeles", "lat": 33.9725, "lng": -118.1547, "conflict_type": "Rear-end", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 13.8}, - {"street_name": "Valley Blvd & Peck Rd", "county": "Los Angeles", "lat": 34.0752, "lng": -118.0277, "conflict_type": "Lane-change", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 12.5}, - {"street_name": "Foothill Blvd & Towne Ave", "county": "Los Angeles", "lat": 34.1238, "lng": -117.6477, "conflict_type": "Crossing", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.2}, - {"street_name": "Indian Ave & I-10", "county": "Riverside", "lat": 33.8680, "lng": -116.5092, "conflict_type": "Lane-change", "min_ttc": 0.7, "min_pet": 1.2, "max_speed": 15.0}, - {"street_name": "Magnolia Ave & Adams St", "county": "Riverside", "lat": 33.9568, "lng": -117.3657, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.4, "max_speed": 12.8}, - {"street_name": "Holt Blvd & Garey Ave", "county": "Los Angeles", "lat": 34.0590, "lng": -117.7513, "conflict_type": "Crossing", "min_ttc": 0.6, "min_pet": 1.0, "max_speed": 14.5}, - {"street_name": "Katella Ave & State College Blvd", "county": "Orange", "lat": 33.8056, "lng": -117.8847, "conflict_type": "Rear-end", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.2}, - {"street_name": "Brookhurst St & Adams Ave", "county": "Orange", "lat": 33.7767, "lng": -117.9380, "conflict_type": "Lane-change", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5}, - {"street_name": "Tustin St & Edinger Ave", "county": "Orange", "lat": 33.7795, "lng": -117.8175, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 13.0}, - {"street_name": "Vista Way & Melrose Dr", "county": "San Diego", "lat": 33.2001, "lng": -117.2412, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5}, - {"street_name": "Mission Ave & College Blvd", "county": "San Diego", "lat": 33.1169, "lng": -117.1580, "conflict_type": "Rear-end", "min_ttc": 1.2, "min_pet": 2.3, "max_speed": 10.5}, - {"street_name": "Palomar St & Broadway", "county": "San Diego", "lat": 32.6224, "lng": -117.0887, "conflict_type": "Lane-change", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.5}, - # Rural / mountain / desert - {"street_name": "Tehachapi Blvd & Tucker Rd", "county": "Kern", "lat": 35.1335, "lng": -118.4498, "conflict_type": "Crossing", "min_ttc": 1.5, "min_pet": 3.5, "max_speed": 8.0}, - {"street_name": "Grass Valley Hwy & Combie Rd", "county": "Nevada", "lat": 39.2019, "lng": -121.0603, "conflict_type": "Rear-end", "min_ttc": 1.4, "min_pet": 3.2, "max_speed": 8.5}, - {"street_name": "Main St & Quincy Junction Rd", "county": "Plumas", "lat": 39.9369, "lng": -120.9466, "conflict_type": "Crossing", "min_ttc": 1.6, "min_pet": 4.0, "max_speed": 7.0}, - {"street_name": "Lassen Ave & Antelope Blvd", "county": "Tehama", "lat": 40.1725, "lng": -122.2393, "conflict_type": "Rear-end", "min_ttc": 1.5, "min_pet": 3.8, "max_speed": 7.5}, - {"street_name": "Main St & Center St", "county": "Lassen", "lat": 40.1767, "lng": -120.6359, "conflict_type": "Crossing", "min_ttc": 1.6, "min_pet": 4.2, "max_speed": 7.0}, - {"street_name": "Main St & S Fir St", "county": "Modoc", "lat": 41.4483, "lng": -120.5419, "conflict_type": "Rear-end", "min_ttc": 1.7, "min_pet": 4.5, "max_speed": 6.5}, - {"street_name": "N Mt Shasta Blvd & Chestnut St", "county": "Siskiyou", "lat": 41.3148, "lng": -122.3098, "conflict_type": "Lane-change", "min_ttc": 1.5, "min_pet": 3.5, "max_speed": 8.0}, - {"street_name": "W Sacramento Ave & Jefferson Blvd", "county": "Yolo", "lat": 38.5819, "lng": -121.5307, "conflict_type": "Rear-end", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5}, - {"street_name": "Elk Grove Blvd & Laguna Blvd", "county": "Sacramento", "lat": 38.4088, "lng": -121.3716, "conflict_type": "Crossing", "min_ttc": 0.8, "min_pet": 1.3, "max_speed": 13.0}, - {"street_name": "Sunrise Blvd & Greenback Ln", "county": "Sacramento", "lat": 38.6764, "lng": -121.2744, "conflict_type": "Lane-change", "min_ttc": 0.7, "min_pet": 1.1, "max_speed": 14.0}, - {"street_name": "Lone Tree Blvd & Main St", "county": "Contra Costa", "lat": 37.9870, "lng": -121.7832, "conflict_type": "Crossing", "min_ttc": 1.0, "min_pet": 1.8, "max_speed": 11.5}, - {"street_name": "Ygnacio Valley Rd & Bancroft Rd", "county": "Contra Costa", "lat": 37.9270, "lng": -122.0001, "conflict_type": "Rear-end", "min_ttc": 1.2, "min_pet": 2.3, "max_speed": 10.5}, - {"street_name": "Santa Rosa Ave & Todd Rd", "county": "Sonoma", "lat": 38.3972, "lng": -122.7055, "conflict_type": "Lane-change", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 11.0}, - {"street_name": "Cleveland Ave & Piner Rd", "county": "Sonoma", "lat": 38.4559, "lng": -122.7455, "conflict_type": "Crossing", "min_ttc": 0.9, "min_pet": 1.5, "max_speed": 12.0}, - {"street_name": "S China Lake Blvd & W Ridgecrest Blvd", "county": "Kern", "lat": 35.6226, "lng": -117.6709, "conflict_type": "Crossing", "min_ttc": 1.3, "min_pet": 2.8, "max_speed": 9.5}, - {"street_name": "Ventura Rd & 5th St", "county": "Ventura", "lat": 34.2022, "lng": -119.1776, "conflict_type": "Crossing", "min_ttc": 1.1, "min_pet": 2.0, "max_speed": 10.8}, -] - -# Add an id and manual_annotation to each record -for _i, _rec in enumerate(_CA_INTERSECTIONS): - _rec["id"] = f"CA-SSAM-{_i+1:04d}" - _rec.setdefault("manual_annotation", "") - - -def _compute_severity(ttc: float, pet: float, max_speed: float) -> dict: - """Return severity_index (0-1) and severity_label.""" - score = 1.0 - (min(ttc, 1.5) / 1.5) * 0.5 - (min(pet, 5.0) / 5.0) * 0.2 - (1.0 - min(max_speed, 18.0) / 18.0) * 0.3 - score = round(max(0.0, min(1.0, score)), 3) - if score >= 0.7: - label = "Critical" - elif score >= 0.45: - label = "High" - elif score >= 0.25: - label = "Medium" - else: - label = "Low" - return {"severity_index": score, "severity_label": label} - - -def load_streets_db(): - """Backwards-compatible loader; returns the old 3-row file if it exists.""" - if not SSAM_STREETS_PATH.exists(): - default_streets = [ - {"street_name": "Market St & 4th St", "min_ttc": 0.8, "min_pet": 1.2, "max_speed": 12.5, "conflict_type": "Crossing", "manual_annotation": ""}, - {"street_name": "Broadway Blvd", "min_ttc": 1.6, "min_pet": 4.5, "max_speed": 8.0, "conflict_type": "Rear-end", "manual_annotation": "High rear-end conflict frequency during commute hours"}, - {"street_name": "Mission St & 16th St", "min_ttc": 0.5, "min_pet": 0.9, "max_speed": 15.0, "conflict_type": "Lane-change", "manual_annotation": ""}, - ] - with open(SSAM_STREETS_PATH, "w") as f: - json.dump(default_streets, f, indent=2) - return default_streets - try: - with open(SSAM_STREETS_PATH, "r") as f: - return json.load(f) - except Exception: - return [] - - -# Legacy endpoint – kept for backward compatibility -@app.get("/api/ssam/streets") -def get_ssam_streets(): - streets = load_streets_db() - for st in streets: - ttc = st.get("min_ttc", 1.5) - pet = st.get("min_pet", 5.0) - score = 1.0 - (min(ttc, 1.5) / 1.5) * 0.7 - (min(pet, 5.0) / 5.0) * 0.3 - st["severity_index"] = round(max(0.0, min(1.0, score)), 2) - return {"status": "ok", "streets": streets} - - -# ----------------------------------------------------------------------- -# NEW: Statewide endpoint with filtering + pagination + map points -# ----------------------------------------------------------------------- -class StatewideQuery(BaseModel): - counties: Optional[List[str]] = None - conflict_types: Optional[List[str]] = None - severity_labels: Optional[List[str]] = None - ttc_max: Optional[float] = None - speed_min: Optional[float] = None - page: int = 1 - page_size: int = 25 - sort_by: str = "severity_index" - sort_dir: str = "desc" - search: Optional[str] = None - - -@app.post("/api/ssam/statewide") -def get_ssam_statewide(q: StatewideQuery): - """Return filtered, paginated SSAM records + GeoJSON points for the map.""" - # Compute severity for every record - enriched = [] - for rec in _CA_INTERSECTIONS: - sev = _compute_severity(rec["min_ttc"], rec["min_pet"], rec["max_speed"]) - row = {**rec, **sev} - enriched.append(row) - - # Apply filters - filtered = enriched - if q.counties: - lc = [c.lower() for c in q.counties] - filtered = [r for r in filtered if r["county"].lower() in lc] - if q.conflict_types: - lc = [c.lower() for c in q.conflict_types] - filtered = [r for r in filtered if r["conflict_type"].lower() in lc] - if q.severity_labels: - lc = [c.lower() for c in q.severity_labels] - filtered = [r for r in filtered if r["severity_label"].lower() in lc] - if q.ttc_max is not None: - filtered = [r for r in filtered if r["min_ttc"] <= q.ttc_max] - if q.speed_min is not None: - filtered = [r for r in filtered if r["max_speed"] >= q.speed_min] - if q.search: - term = q.search.lower() - filtered = [r for r in filtered if term in r["street_name"].lower() or term in r["county"].lower()] - - # Sort - reverse = q.sort_dir.lower() == "desc" - if q.sort_by in ("severity_index", "min_ttc", "min_pet", "max_speed"): - filtered.sort(key=lambda r: r.get(q.sort_by, 0), reverse=reverse) - elif q.sort_by == "county": - filtered.sort(key=lambda r: r.get("county", ""), reverse=reverse) - elif q.sort_by == "street_name": - filtered.sort(key=lambda r: r.get("street_name", ""), reverse=reverse) - - total = len(filtered) - total_pages = max(1, (total + q.page_size - 1) // q.page_size) - page = max(1, min(q.page, total_pages)) - start = (page - 1) * q.page_size - page_rows = filtered[start:start + q.page_size] - - # Build GeoJSON features for ALL filtered records (for the map) - features = [] - for r in filtered: - features.append({ - "type": "Feature", - "geometry": {"type": "Point", "coordinates": [r["lng"], r["lat"]]}, - "properties": { - "id": r["id"], - "street_name": r["street_name"], - "county": r["county"], - "conflict_type": r["conflict_type"], - "min_ttc": r["min_ttc"], - "min_pet": r["min_pet"], - "max_speed": r["max_speed"], - "severity_index": r["severity_index"], - "severity_label": r["severity_label"], - } - }) - - # Summary stats per severity - summary = {"Critical": 0, "High": 0, "Medium": 0, "Low": 0} - for r in filtered: - summary[r["severity_label"]] = summary.get(r["severity_label"], 0) + 1 - - # Unique counties and conflict types (for filter dropdowns) - all_counties = sorted({r["county"] for r in enriched}) - all_conflict_types = sorted({r["conflict_type"] for r in enriched}) - - return { - "status": "ok", - "total": total, - "page": page, - "page_size": q.page_size, - "total_pages": total_pages, - "rows": page_rows, - "geojson": {"type": "FeatureCollection", "features": features}, - "summary": summary, - "filter_options": { - "counties": all_counties, - "conflict_types": all_conflict_types, - "severity_labels": ["Critical", "High", "Medium", "Low"], - }, - } - - -@app.post("/api/ssam/annotate") -def annotate_street(req: AnnotateStreetRequest): - streets = load_streets_db() - found = False - for st in streets: - if st["street_name"] == req.street_name: - st["manual_annotation"] = req.manual_annotation - found = True - break - if not found: - streets.append({ - "street_name": req.street_name, - "min_ttc": 1.5, - "min_pet": 5.0, - "max_speed": 0.0, - "conflict_type": "Rear-end", - "manual_annotation": req.manual_annotation - }) - with open(SSAM_STREETS_PATH, "w") as f: - json.dump(streets, f, indent=2) - return {"status": "ok", "message": f"Annotation saved for {req.street_name}"} - -# ----------------------------------------------------------------------- -# 3D Perception Pipeline Routes -# ----------------------------------------------------------------------- - -class IngestParams(BaseModel): - vendors: List[str] = ["alpamayo", "waymo"] - sequence_id: str = "seq_001" - -class AutoLabelParams(BaseModel): - sequence_id: str = "seq_001" - sam_checkpoint: str = "models/sam_vit_b.pth" - device: str = "cpu" - no_sam: bool = False - -class TrackParams(BaseModel): - sequence_id: str = "seq_001" - -class BenchmarkParams(BaseModel): - sequence_id: str = "seq_001" - -class GateParams(BaseModel): - sequence_id: str = "seq_001" - - -@app.post("/api/dataset/ingest") -def ingest_dataset(params: IngestParams): - from sensorflow.dataset_fusion_engine import DatasetFusionEngine - try: - engine = DatasetFusionEngine() - sequence = engine.ingest(params.vendors, params.sequence_id) - manifest_path = engine.save_manifest(sequence) - except Exception as e: - raise HTTPException(status_code=500, detail=f"Ingest failed: {str(e)}") - - config = load_config() - config.sequence_id = params.sequence_id - config.vendors = params.vendors - with open(CONFIG_PATH, "w") as f: - json.dump(config.model_dump(), f, indent=2) - - return {"status": "ok", "manifest": str(manifest_path), "sequence_id": params.sequence_id} - - -@app.get("/api/dataset/ingest/status") -def ingest_status(sequence_id: str = "seq_001"): - from sensorflow.dataset_fusion_engine import DatasetFusionEngine - engine = DatasetFusionEngine() - status = engine.get_status(sequence_id) - manifest = Path("runs/pipeline") / sequence_id / "manifest.json" - if manifest.exists(): - with open(manifest) as f: - manifest_data = json.load(f) - status["manifest_exists"] = True - status["frames"] = len(manifest_data.get("frames", [])) - status["vendor"] = manifest_data.get("vendor", "unknown") - return {"status": "ok", "sequence_id": sequence_id, **status} - - -@app.post("/api/perception/auto-label") -def auto_label(params: AutoLabelParams): - from sensorflow.perception_automator import PerceptionAutomator - from sensorflow.schemas.unified_frame import UnifiedSequence - - manifest = Path("runs/pipeline") / params.sequence_id / "manifest.json" - if not manifest.exists(): - raise HTTPException(status_code=400, detail="Manifest not found. Run ingest first.") - - try: - sequence = UnifiedSequence.load(manifest) - output_dir = manifest.parent / "proposals" - automator = PerceptionAutomator( - sam_checkpoint=params.sam_checkpoint, - device=params.device, - use_sam=not params.no_sam, - ) - automator.run_sequence(sequence, output_dir) - except Exception as e: - raise HTTPException(status_code=500, detail=f"Auto-label failed: {str(e)}") - - proposals_dir = manifest.parent / "proposals" - num_frames = len(list(proposals_dir.glob("*.json"))) if proposals_dir.exists() else 0 - return {"status": "ok", "proposals_dir": str(proposals_dir), "frames_processed": num_frames} - - -@app.post("/api/perception/track") -def run_tracking(params: TrackParams): - from sensorflow.temporal_tracker import TemporalTracker - - proposals_dir = Path("runs/pipeline") / params.sequence_id / "proposals" - if not proposals_dir.exists(): - raise HTTPException(status_code=400, detail="Proposals not found. Run auto-label first.") - - output = Path("runs/pipeline") / params.sequence_id / "tracks.json" - try: - tracker = TemporalTracker() - proposals = TemporalTracker.load_proposals(proposals_dir) - tracks = tracker.run_sequence(proposals, output) - except Exception as e: - raise HTTPException(status_code=500, detail=f"Tracking failed: {str(e)}") - - return {"status": "ok", "tracks_file": str(output), "num_tracks": len(tracks)} - - -@app.post("/api/benchmark") -def run_benchmark(params: BenchmarkParams): - from sensorflow.quality_gate import QualityGate - from sensorflow.schemas.unified_frame import UnifiedSequence - - manifest = Path("runs/pipeline") / params.sequence_id / "manifest.json" - tracks_path = Path("runs/pipeline") / params.sequence_id / "tracks.json" - if not manifest.exists(): - raise HTTPException(status_code=400, detail="Manifest not found.") - - try: - sequence = UnifiedSequence.load(manifest) - pred_tracks = json.loads(tracks_path.read_text()) if tracks_path.exists() else [] - gate = QualityGate() - results = gate.evaluate(sequence, pred_tracks) - gate.save_results(params.sequence_id, results) - except Exception as e: - raise HTTPException(status_code=500, detail=f"Benchmark failed: {str(e)}") - - return {"status": "ok", "metrics": results["metric_card"]} - - -@app.post("/api/gates/quality") -def quality_gate_eval(params: GateParams): - from sensorflow.quality_gate import QualityGate - from sensorflow.mitl_copilot import MitlCopilot - from sensorflow.schemas.unified_frame import UnifiedSequence - - manifest = Path("runs/pipeline") / params.sequence_id / "manifest.json" - tracks_path = Path("runs/pipeline") / params.sequence_id / "tracks.json" - if not manifest.exists() or not tracks_path.exists(): - raise HTTPException(status_code=400, detail="Run ingest, auto-label, and track first.") - - try: - sequence = UnifiedSequence.load(manifest) - with open(tracks_path) as f: - pred_tracks = json.load(f) - gate = QualityGate() - results = gate.evaluate(sequence, pred_tracks) - gate.save_results(params.sequence_id, results) - if not results["passed"]: - copilot = MitlCopilot() - copilot.route_edge_cases(params.sequence_id, results["metric_card"], pred_tracks) - except Exception as e: - raise HTTPException(status_code=500, detail=f"Quality gate failed: {str(e)}") - - report = results["quality_report"] - return {"status": "ok", "metric_card": results["metric_card"], **report} - - -@app.post("/api/gates/launch") -def launch_gate_eval(params: GateParams): - from sensorflow.launch_gate_evaluator import LaunchGateEvaluator - config = load_config() - evaluator = LaunchGateEvaluator(Path(config.gate_thresholds_path)) - result = evaluator.evaluate(params.sequence_id) - return {"status": "ok", **result} - - -@app.get("/api/pipeline/status") -def pipeline_status(sequence_id: str = "seq_001"): - state_path = Path("runs/pipeline/state.json") - state = {} - if state_path.exists(): - with open(state_path) as f: - state = json.load(f) - - seq_state = state.get(sequence_id, {}) - base = Path("runs/pipeline") / sequence_id - return { - "status": "ok", - "sequence_id": sequence_id, - "ingest_complete": (base / "manifest.json").exists(), - "perception_complete": (base / "proposals").exists() and any((base / "proposals").glob("*.json")), - "tracking_complete": (base / "tracks.json").exists(), - "benchmark_complete": (base / "benchmark" / "metric_card.json").exists(), - "launch_gate_passed": seq_state.get("launch_gate_passed", False), - "state": seq_state, - } - - -@app.get("/api/mitl/queue") -def get_mitl_queue(sequence_id: str = "seq_001"): - from sensorflow.mitl_copilot import MitlCopilot - copilot = MitlCopilot() - return {"status": "ok", "queue": copilot.get_queue(sequence_id)} - - -# L4 Perception Label Evaluation platform routes (must precede the static mount). -from sensorflow.evaluation.api import router as labeleval_router -app.include_router(labeleval_router) - -# Aggregate-first mega-scale evaluation layer (metric cube, async runs, query API). -from sensorflow.megaeval.api import router as megaeval_router -app.include_router(megaeval_router) - -# BEV-Fusion perception engine (camera+LiDAR fusion, masklet tracking, self-eval). -from sensorflow.bevfusion.api import router as bevfusion_router -app.include_router(bevfusion_router) - -# Mount static folder -static_dir = Path(__file__).parent / "static" -if static_dir.exists(): - app.mount("/", StaticFiles(directory=str(static_dir), html=True), name="static") - -if __name__ == "__main__": - import uvicorn - uvicorn.run("app_backend:app", host="127.0.0.1", port=8000, reload=True) diff --git a/hf-space/autograder.py b/hf-space/autograder.py deleted file mode 100644 index 0a306fc5aae4f0363994de6ec69c932578cf64de..0000000000000000000000000000000000000000 --- a/hf-space/autograder.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python3 -"""2D prediction quality autograder.""" - -import argparse -import json -from collections import Counter -from pathlib import Path -from typing import Dict, List - - -def compute_iou(box_a: Dict, box_b: Dict) -> float: - x1 = max(box_a["x1"], box_b["x1"]) - y1 = max(box_a["y1"], box_b["y1"]) - x2 = min(box_a["x2"], box_b["x2"]) - y2 = min(box_a["y2"], box_b["y2"]) - inter = max(0, x2 - x1) * max(0, y2 - y1) - area_a = (box_a["x2"] - box_a["x1"]) * (box_a["y2"] - box_a["y1"]) - area_b = (box_b["x2"] - box_b["x1"]) * (box_b["y2"] - box_b["y1"]) - union = area_a + area_b - inter - return inter / union if union > 0 else 0.0 - - -def grade_predictions(predictions: List[Dict], conf_threshold: float = 0.5) -> Dict: - issues = [] - issue_summary = Counter() - images = set(p["image"] for p in predictions) - classes = [p["class_name"] for p in predictions] - - for i, pred in enumerate(predictions): - if pred["confidence"] < conf_threshold: - issue_summary["low_confidence"] += 1 - issues.append({ - "severity": "WARNING", - "type": "low_confidence", - "image": pred["image"], - "detection_id": i, - "confidence": pred["confidence"], - "threshold": conf_threshold, - "message": f"Confidence {pred['confidence']:.2f} below threshold", - }) - - bbox = pred["bbox"] - area = (bbox["x2"] - bbox["x1"]) * (bbox["y2"] - bbox["y1"]) - if area < 100: - issue_summary["small_detection"] += 1 - issues.append({ - "severity": "INFO", - "type": "small_detection", - "image": pred["image"], - "detection_id": i, - "confidence": pred["confidence"], - "threshold": 100, - "message": f"Small bounding box area: {area:.0f}px", - }) - - for i, pred_a in enumerate(predictions): - for j, pred_b in enumerate(predictions): - if j <= i or pred_a["image"] != pred_b["image"]: - continue - if pred_a["class_name"] != pred_b["class_name"]: - iou = compute_iou(pred_a["bbox"], pred_b["bbox"]) - if iou > 0.5: - issue_summary["overlapping_different_class"] += 1 - issues.append({ - "severity": "WARNING", - "type": "overlapping_different_class", - "image": pred_a["image"], - "detection_id": i, - "confidence": pred_a["confidence"], - "threshold": 0.5, - "message": f"Overlap between {pred_a['class_name']} and {pred_b['class_name']}", - }) - - class_counts = Counter(classes) - if class_counts: - dominant = class_counts.most_common(1)[0] - if dominant[1] / len(classes) > 0.8: - issue_summary["class_imbalance"] += 1 - - total_issues = sum(issue_summary.values()) - quality_score = max(0, 100 - total_issues * 5) - - return { - "total_predictions": len(predictions), - "total_images": len(images), - "quality_score": quality_score, - "issues_by_severity": dict(Counter(i["severity"] for i in issues)), - "issue_summary": dict(issue_summary), - "flagged_issues": issues[:50], - "recommendations": _recommendations(issue_summary), - } - - -def _recommendations(summary: Counter) -> List[str]: - recs = [] - if summary.get("low_confidence"): - recs.append("Tune confidence threshold or retrain with more diverse samples.") - if summary.get("overlapping_different_class"): - recs.append("Review overlapping class labels; consider NMS tuning.") - if summary.get("small_detection"): - recs.append("Verify small bounding boxes are valid annotations.") - if summary.get("class_imbalance"): - recs.append("Balance class frequency in training dataset.") - if not recs: - recs.append("No significant issues detected.") - return recs - - -def main(): - parser = argparse.ArgumentParser(description="Grade 2D predictions") - parser.add_argument("--predictions", required=True) - parser.add_argument("--output", default=None) - parser.add_argument("--conf", type=float, default=0.5) - args = parser.parse_args() - - pred_path = Path(args.predictions) - with open(pred_path) as f: - predictions = json.load(f) - - report = grade_predictions(predictions, args.conf) - output_path = Path(args.output) if args.output else pred_path.parent / "quality_report.json" - with open(output_path, "w") as f: - json.dump(report, f, indent=2) - print(f"Quality score: {report['quality_score']}/100 -> {output_path}") - - -if __name__ == "__main__": - main() diff --git a/hf-space/coco8.yaml b/hf-space/coco8.yaml deleted file mode 100644 index ef0725380fd3d1645eaa44a36c4b0cca38401aac..0000000000000000000000000000000000000000 --- a/hf-space/coco8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Ultralytics YOLO 🚀, AGPL-3.0 license -# COCO8 dataset by Ultralytics -# Documentation: https://docs.ultralytics.com/datasets/detect/coco8/ - -# Dataset root directory -path: ./datasets/coco8 - -# Train and val images -train: images/train -val: images/val - -# Classes -names: - 0: person - 1: bicycle - 2: car - 3: motorcycle - 4: airplane - 5: bus - 6: train - 7: truck diff --git a/hf-space/comparative_analytics.py b/hf-space/comparative_analytics.py deleted file mode 100644 index 2c0a22bf3627679e0b800b08648e21fc8466a5cc..0000000000000000000000000000000000000000 --- a/hf-space/comparative_analytics.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Comparative Analytics Dashboard - Core Engine""" -import pandas as pd -import numpy as np -from typing import Dict, List, Optional, Tuple, Any -from dataclasses import dataclass, field, asdict -from enum import Enum -import json -from datetime import datetime - - -class MetricType(Enum): - """Metric categories""" - ACCURACY = "accuracy" - F1_SCORE = "f1_score" - AUC = "auc" - INFERENCE_TIME = "inference_time" - MEMORY_USAGE = "memory_usage" - - -@dataclass -class Configuration: - """Configuration parameters""" - config_id: str - name: str - technique: str - parameters: Dict[str, Any] - def __hash__(self): - return hash(self.config_id) - - -@dataclass -class PerformanceMetrics: - """Evaluation metrics""" - accuracy: Optional[float] = None - precision: Optional[float] = None - recall: Optional[float] = None - f1_score: Optional[float] = None - auc: Optional[float] = None - inference_time: Optional[float] = None - memory_usage: Optional[float] = None - training_time: Optional[float] = None - throughput: Optional[float] = None - latency: Optional[float] = None - - def to_dict(self) -> Dict[str, Optional[float]]: - return {k: v for k, v in asdict(self).items() if v is not None} - - def get_metric(self, metric_name: str) -> Optional[float]: - return getattr(self, metric_name, None) - - -@dataclass -class Perception3DMetrics(PerformanceMetrics): - """3D perception and temporal tracking metrics.""" - map_3d: Optional[float] = None - mar_3d: Optional[float] = None - mean_iou_3d: Optional[float] = None - orientation_error_deg: Optional[float] = None - position_error_m: Optional[float] = None - id_swap_rate: Optional[float] = None - track_fragmentation_rate: Optional[float] = None - process_units: Optional[int] = None - compute_cycles: Optional[int] = None - - @classmethod - def from_metric_card(cls, card: Dict[str, Any]) -> "Perception3DMetrics": - return cls( - map_3d=card.get("map_3d"), - mar_3d=card.get("mar_3d"), - mean_iou_3d=card.get("mean_iou_3d"), - orientation_error_deg=card.get("orientation_error_deg"), - position_error_m=card.get("position_error_m"), - id_swap_rate=card.get("id_swap_rate"), - track_fragmentation_rate=card.get("track_fragmentation_rate"), - process_units=card.get("process_units"), - compute_cycles=card.get("compute_cycles"), - ) - - -@dataclass -class ExperimentResult: - """Single experiment result""" - dataset: str - model: str - configuration: Configuration - metrics: PerformanceMetrics - timestamp: str = field(default_factory=lambda: datetime.now().isoformat()) - notes: str = "" - - def __hash__(self): - return hash((self.dataset, self.model, self.configuration.config_id)) - - -class ComparativeAnalytics: - """Analytics engine for comparing results""" - - def __init__(self): - self.results: List[ExperimentResult] = [] - self.datasets: set = set() - self.models: set = set() - self.techniques: set = set() - self.configurations: Dict[str, Configuration] = {} - - def add_result(self, result: ExperimentResult) -> None: - self.results.append(result) - self.datasets.add(result.dataset) - self.models.add(result.model) - self.techniques.add(result.configuration.technique) - self.configurations[result.configuration.config_id] = result.configuration - - def add_results_batch(self, results: List[ExperimentResult]) -> None: - for result in results: - self.add_result(result) - - def filter_results(self, datasets: Optional[List[str]] = None, models: Optional[List[str]] = None, - techniques: Optional[List[str]] = None, configurations: Optional[List[str]] = None) -> List[ExperimentResult]: - filtered = self.results - if datasets: - filtered = [r for r in filtered if r.dataset in datasets] - if models: - filtered = [r for r in filtered if r.model in models] - if techniques: - filtered = [r for r in filtered if r.configuration.technique in techniques] - if configurations: - filtered = [r for r in filtered if r.configuration.config_id in configurations] - return filtered - - def to_dataframe(self, datasets: Optional[List[str]] = None, models: Optional[List[str]] = None, - techniques: Optional[List[str]] = None, configurations: Optional[List[str]] = None) -> pd.DataFrame: - filtered = self.filter_results(datasets, models, techniques, configurations) - rows = [] - for result in filtered: - row = { - 'dataset': result.dataset, - 'model': result.model, - 'technique': result.configuration.technique, - 'configuration': result.configuration.name, - 'config_id': result.configuration.config_id, - 'timestamp': result.timestamp, - } - row.update(result.metrics.to_dict()) - rows.append(row) - return pd.DataFrame(rows) - - def get_best_configurations(self, metric: str = 'f1_score', dataset: Optional[str] = None, - model: Optional[str] = None, top_n: int = 5) -> pd.DataFrame: - filters = {'datasets': [dataset] if dataset else None, 'models': [model] if model else None} - df = self.to_dataframe(**filters) - if df.empty or metric not in df.columns: - return pd.DataFrame() - df = df.dropna(subset=[metric]) - df = df.sort_values(metric, ascending=False).head(top_n) - return df - - def compare_techniques(self, metric: str = 'f1_score', dataset: Optional[str] = None, - model: Optional[str] = None) -> pd.DataFrame: - filters = {'datasets': [dataset] if dataset else None, 'models': [model] if model else None} - df = self.to_dataframe(**filters) - if df.empty or metric not in df.columns: - return pd.DataFrame() - comparison = df.groupby('technique')[metric].agg([('mean', 'mean'), ('std', 'std'), - ('min', 'min'), ('max', 'max'), ('count', 'count')]).round(4) - return comparison.sort_values('mean', ascending=False) - - def get_summary_statistics(self, metric: str = 'f1_score', groupby: str = 'technique') -> Dict: - df = self.to_dataframe() - if df.empty or metric not in df.columns: - return {} - stats = {} - for group_name, group_df in df.groupby(groupby): - group_data = group_df[metric].dropna() - stats[str(group_name)] = { - 'mean': float(group_data.mean()), - 'std': float(group_data.std()), - 'min': float(group_data.min()), - 'max': float(group_data.max()), - 'median': float(group_data.median()), - 'count': int(len(group_data)) - } - return stats - - def get_dataset_model_matrix(self, metric: str = 'f1_score') -> pd.DataFrame: - df = self.to_dataframe() - if df.empty or metric not in df.columns: - return pd.DataFrame() - matrix = df.groupby(['dataset', 'model'])[metric].mean().unstack(fill_value=None) - return matrix - - -class DashboardBuilder: - """Build dashboard components""" - def __init__(self, analytics: ComparativeAnalytics): - self.analytics = analytics - - def get_summary_report(self) -> Dict[str, Any]: - df = self.analytics.to_dataframe() - if df.empty: - return {} - return { - 'overview': { - 'total_experiments': len(self.analytics.results), - 'num_datasets': len(self.analytics.datasets), - 'num_models': len(self.analytics.models), - 'num_techniques': len(self.analytics.techniques), - }, - 'datasets': sorted(list(self.analytics.datasets)), - 'models': sorted(list(self.analytics.models)), - 'techniques': sorted(list(self.analytics.techniques)), - } diff --git a/hf-space/dashboard_viz.py b/hf-space/dashboard_viz.py deleted file mode 100644 index baecf637ed5d9e4034fcc8f62fc5b01ea8a9b74b..0000000000000000000000000000000000000000 --- a/hf-space/dashboard_viz.py +++ /dev/null @@ -1,468 +0,0 @@ -""" -Dashboard Visualizations - Complete Implementation -Bar charts, heatmaps, scatter plots, and Pareto frontier -""" - -import pandas as pd -import numpy as np -from typing import Dict, List, Optional, Tuple, Any -from comparative_analytics import ComparativeAnalytics - -try: - import plotly.graph_objects as go - import plotly.express as px - from plotly.subplots import make_subplots - PLOTLY_AVAILABLE = True -except ImportError: - PLOTLY_AVAILABLE = False - print("Warning: plotly not installed. Install with: pip install plotly") - - -class DashboardVisualizations: - """Generate interactive visualizations for comparative analysis""" - - def __init__(self, analytics: ComparativeAnalytics): - self.analytics = analytics - if not PLOTLY_AVAILABLE: - raise ImportError("plotly required for visualizations. Install with: pip install plotly") - - def bar_chart_by_technique(self, metric: str = 'f1_score', dataset: Optional[str] = None, - model: Optional[str] = None) -> go.Figure: - """Bar chart comparing metric values by technique""" - df = self.analytics.to_dataframe( - datasets=[dataset] if dataset else None, - models=[model] if model else None - ) - - if df.empty or metric not in df.columns: - return go.Figure().add_annotation(text="No data available") - - # Group by technique and calculate statistics - tech_stats = df.groupby('technique')[metric].agg(['mean', 'std']).reset_index() - tech_stats = tech_stats.sort_values('mean', ascending=False) - - fig = go.Figure() - - fig.add_trace(go.Bar( - x=tech_stats['technique'], - y=tech_stats['mean'], - error_y=dict(type='data', array=tech_stats['std'], visible=True), - name='Mean ± Std', - marker=dict( - color=tech_stats['mean'], - colorscale='Viridis', - showscale=True, - colorbar=dict(title=metric) - ), - text=tech_stats['mean'].round(4), - textposition='auto', - )) - - title = f"{metric.replace('_', ' ').title()} by Technique" - if dataset: - title += f" ({dataset})" - if model: - title += f" ({model})" - - fig.update_layout( - title=title, - xaxis_title='Technique', - yaxis_title=metric.replace('_', ' ').title(), - hovermode='x unified', - height=500, - showlegend=True - ) - - return fig - - def bar_chart_by_configuration(self, metric: str = 'f1_score', technique: Optional[str] = None, - top_n: int = 10) -> go.Figure: - """Bar chart showing top N configurations""" - df = self.analytics.to_dataframe( - techniques=[technique] if technique else None - ) - - if df.empty or metric not in df.columns: - return go.Figure().add_annotation(text="No data available") - - # Get top configurations - top = df.nlargest(top_n, metric)[['configuration', 'technique', metric]] - - fig = go.Figure() - - fig.add_trace(go.Bar( - y=top['configuration'], - x=top[metric], - orientation='h', - marker=dict( - color=top[metric], - colorscale='Viridis', - showscale=True, - colorbar=dict(title=metric) - ), - text=top[metric].round(4), - textposition='auto', - hovertemplate='%{y}
    ' + f'{metric}: %{{x:.4f}}' - )) - - title = f"Top {top_n} Configurations by {metric.replace('_', ' ').title()}" - if technique: - title += f" ({technique})" - - fig.update_layout( - title=title, - xaxis_title=metric.replace('_', ' ').title(), - yaxis_title='Configuration', - height=400 + top_n * 20, - showlegend=False - ) - - return fig - - def heatmap_dataset_technique(self, metric: str = 'f1_score') -> go.Figure: - """Heatmap: Dataset × Technique""" - df = self.analytics.to_dataframe() - - if df.empty or metric not in df.columns: - return go.Figure().add_annotation(text="No data available") - - # Create pivot table - pivot = df.pivot_table( - values=metric, - index='dataset', - columns='technique', - aggfunc='mean' - ) - - fig = go.Figure(data=go.Heatmap( - z=pivot.values, - x=pivot.columns, - y=pivot.index, - colorscale='RdYlGn', - text=np.round(pivot.values, 4), - texttemplate='%{text:.4f}', - textfont={"size": 10}, - hovertemplate='Dataset: %{y}
    Technique: %{x}
    ' + f'{metric}: %{{z:.4f}}', - colorbar=dict(title=metric) - )) - - fig.update_layout( - title=f"{metric.replace('_', ' ').title()} Heatmap: Dataset × Technique", - xaxis_title='Technique', - yaxis_title='Dataset', - height=400, - width=700 - ) - - return fig - - def heatmap_dataset_model(self, metric: str = 'f1_score') -> go.Figure: - """Heatmap: Dataset × Model""" - df = self.analytics.to_dataframe() - - if df.empty or metric not in df.columns: - return go.Figure().add_annotation(text="No data available") - - # Create pivot table - pivot = df.pivot_table( - values=metric, - index='dataset', - columns='model', - aggfunc='mean' - ) - - fig = go.Figure(data=go.Heatmap( - z=pivot.values, - x=pivot.columns, - y=pivot.index, - colorscale='RdYlGn', - text=np.round(pivot.values, 4), - texttemplate='%{text:.4f}', - textfont={"size": 10}, - hovertemplate='Dataset: %{y}
    Model: %{x}
    ' + f'{metric}: %{{z:.4f}}', - colorbar=dict(title=metric) - )) - - fig.update_layout( - title=f"{metric.replace('_', ' ').title()} Heatmap: Dataset × Model", - xaxis_title='Model', - yaxis_title='Dataset', - height=400, - width=700 - ) - - return fig - - def scatter_plot_trade_off(self, metric_x: str = 'f1_score', metric_y: str = 'inference_time', - color_by: str = 'technique') -> go.Figure: - """Scatter plot for two-metric trade-off""" - df = self.analytics.to_dataframe() - - if df.empty or metric_x not in df.columns or metric_y not in df.columns: - return go.Figure().add_annotation(text="No data available") - - df = df.dropna(subset=[metric_x, metric_y]) - - fig = px.scatter( - df, - x=metric_x, - y=metric_y, - color=color_by, - hover_data=['dataset', 'model', 'configuration'], - title=f"Trade-off: {metric_x.replace('_', ' ').title()} vs {metric_y.replace('_', ' ').title()}", - labels={ - metric_x: metric_x.replace('_', ' ').title(), - metric_y: metric_y.replace('_', ' ').title() - } - ) - - fig.update_layout( - height=500, - hovermode='closest' - ) - - return fig - - def line_chart_trends(self, metric: str = 'f1_score', technique: Optional[str] = None, - group_by: str = 'configuration') -> go.Figure: - """Line chart showing performance trends""" - df = self.analytics.to_dataframe( - techniques=[technique] if technique else None - ) - - if df.empty or metric not in df.columns: - return go.Figure().add_annotation(text="No data available") - - df = df.sort_values('timestamp') - - fig = go.Figure() - - # Line for each configuration - for config in df[group_by].unique(): - config_df = df[df[group_by] == config] - fig.add_trace(go.Scatter( - x=config_df.index, - y=config_df[metric], - mode='lines+markers', - name=config, - hovertemplate='%{fullData.name}
    ' + - f'{metric}: %{{y:.4f}}
    ' + - 'Index: %{x}' - )) - - title = f"{metric.replace('_', ' ').title()} Trends" - if technique: - title += f" ({technique})" - - fig.update_layout( - title=title, - xaxis_title='Experiment Index', - yaxis_title=metric.replace('_', ' ').title(), - hovermode='x unified', - height=500 - ) - - return fig - - def pareto_frontier_plot(self, metric1: str = 'f1_score', metric2: str = 'inference_time') -> go.Figure: - """Plot showing Pareto frontier (optimal trade-offs)""" - df = self.analytics.to_dataframe() - - if df.empty or metric1 not in df.columns or metric2 not in df.columns: - return go.Figure().add_annotation(text="No data available") - - df = df.dropna(subset=[metric1, metric2]) - - # Calculate Pareto frontier - pareto_mask = np.ones(len(df), dtype=bool) - for i in range(len(df)): - for j in range(len(df)): - if i != j: - # Check if point j dominates point i - if df.iloc[j][metric1] >= df.iloc[i][metric1] and \ - df.iloc[j][metric2] <= df.iloc[i][metric2] and \ - (df.iloc[j][metric1] > df.iloc[i][metric1] or df.iloc[j][metric2] < df.iloc[i][metric2]): - pareto_mask[i] = False - break - - pareto_df = df[pareto_mask] - - fig = go.Figure() - - # All points - fig.add_trace(go.Scatter( - x=df[metric2], - y=df[metric1], - mode='markers', - name='All Results', - marker=dict(size=8, color='lightblue', opacity=0.6), - hovertemplate=f'{metric1}: %{{y:.4f}}
    {metric2}: %{{x:.4f}}' - )) - - # Pareto frontier - if not pareto_df.empty: - pareto_sorted = pareto_df.sort_values(metric2) - fig.add_trace(go.Scatter( - x=pareto_sorted[metric2], - y=pareto_sorted[metric1], - mode='lines+markers', - name='Pareto Frontier', - marker=dict(size=12, color='red'), - line=dict(color='red', width=2), - hovertemplate=f'{metric1}: %{{y:.4f}}
    {metric2}: %{{x:.4f}}' - )) - - fig.update_layout( - title=f"Pareto Frontier: {metric1.replace('_', ' ').title()} vs {metric2.replace('_', ' ').title()}", - xaxis_title=metric2.replace('_', ' ').title(), - yaxis_title=metric1.replace('_', ' ').title(), - hovermode='closest', - height=500 - ) - - return fig - - def multi_metric_subplots(self, metrics: List[str] = ['f1_score', 'accuracy', 'inference_time']) -> go.Figure: - """Subplots comparing techniques across multiple metrics""" - df = self.analytics.to_dataframe() - - if df.empty: - return go.Figure().add_annotation(text="No data available") - - available_metrics = [m for m in metrics if m in df.columns] - - if not available_metrics: - return go.Figure().add_annotation(text="No metrics available") - - fig = make_subplots( - rows=1, - cols=len(available_metrics), - subplot_titles=[m.replace('_', ' ').title() for m in available_metrics], - specs=[[{"type": "bar"}] * len(available_metrics)] - ) - - colors = ['rgb(31, 119, 180)', 'rgb(255, 127, 14)', 'rgb(44, 160, 44)'] - - for col_idx, metric in enumerate(available_metrics, 1): - df_clean = df.dropna(subset=[metric]) - stats = df_clean.groupby('technique')[metric].mean().sort_values(ascending=False) - - fig.add_trace( - go.Bar( - x=stats.index, - y=stats.values, - name=metric, - marker=dict(color=colors[col_idx % len(colors)]), - text=stats.round(4), - textposition='auto', - ), - row=1, - col=col_idx - ) - - fig.update_xaxes(title_text='Technique', row=1, col=col_idx) - fig.update_yaxes(title_text=metric.replace('_', ' ').title(), row=1, col=col_idx) - - fig.update_layout( - title='Technique Comparison Across Multiple Metrics', - height=500, - showlegend=False - ) - - return fig - - def export_all_charts_html(self, filepath: str, metrics: List[str] = ['f1_score', 'accuracy']) -> str: - """Export all visualizations to a single HTML file""" - html_parts = [ - """ - - - Comparative Analytics Dashboard - - - - -
    -

    📊 Comparative Analytics Dashboard

    - """ - ] - - # Overview - summary = self.analytics.to_dataframe() - if not summary.empty: - html_parts.append(f""" -
    -

    Overview

    -

    Total Experiments: {len(self.analytics.results)}

    -

    Datasets: {', '.join(sorted(self.analytics.datasets))}

    -

    Models: {', '.join(sorted(self.analytics.models))}

    -

    Techniques: {', '.join(sorted(self.analytics.techniques))}

    -
    - """) - - # Visualizations - html_parts.append('

    Visualizations

    ') - - # Bar charts by technique - for metric in metrics: - fig = self.bar_chart_by_technique(metric=metric) - html_parts.append(f'

    {metric.replace("_", " ").title()} by Technique

    ') - html_parts.append(fig.to_html(include_plotlyjs=False, div_id=f"chart_{metric}_tech")) - html_parts.append('
    ') - - # Heatmaps - fig_hm_dt = self.heatmap_dataset_technique() - html_parts.append('

    Dataset × Technique Heatmap (F1-Score)

    ') - html_parts.append(fig_hm_dt.to_html(include_plotlyjs=False, div_id="heatmap_dt")) - html_parts.append('
    ') - - fig_hm_dm = self.heatmap_dataset_model() - html_parts.append('

    Dataset × Model Heatmap (F1-Score)

    ') - html_parts.append(fig_hm_dm.to_html(include_plotlyjs=False, div_id="heatmap_dm")) - html_parts.append('
    ') - - # Scatter plot - fig_scatter = self.scatter_plot_trade_off() - html_parts.append('

    Trade-off Analysis (F1-Score vs Inference Time)

    ') - html_parts.append(fig_scatter.to_html(include_plotlyjs=False, div_id="scatter")) - html_parts.append('
    ') - - # Pareto frontier - fig_pareto = self.pareto_frontier_plot() - html_parts.append('

    Pareto Frontier

    ') - html_parts.append(fig_pareto.to_html(include_plotlyjs=False, div_id="pareto")) - html_parts.append('
    ') - - # Multi-metric subplots - fig_multi = self.multi_metric_subplots(metrics=metrics + ['inference_time']) - html_parts.append('

    Multi-Metric Comparison

    ') - html_parts.append(fig_multi.to_html(include_plotlyjs=False, div_id="subplots")) - html_parts.append('
    ') - - # Summary statistics table - stats = self.analytics.compare_techniques() - if not stats.empty: - html_parts.append('

    Summary Statistics

    ') - html_parts.append(stats.to_html()) - html_parts.append('
    ') - - html_parts.append('
    ') - - html_content = '\n'.join(html_parts) - - with open(filepath, 'w') as f: - f.write(html_content) - - return filepath diff --git a/hf-space/data/test.jpg b/hf-space/data/test.jpg deleted file mode 100644 index eeab1cdcb282b0e026a57c5bf85df36024b4e1f6..0000000000000000000000000000000000000000 Binary files a/hf-space/data/test.jpg and /dev/null differ diff --git a/hf-space/deploy/CHECKLIST.md b/hf-space/deploy/CHECKLIST.md deleted file mode 100644 index b0c979456f90df06b2c9d7d7f54fe46fa2e9370c..0000000000000000000000000000000000000000 --- a/hf-space/deploy/CHECKLIST.md +++ /dev/null @@ -1,140 +0,0 @@ -# Pre-Deployment & Post-Deployment Checklist - -## Pre-Deployment (Local Machine) - -- [x] Cleanup complete: - - [x] Removed test files (test_*.py) - - [x] Removed examples (example_*.py) - - [x] Removed virtual environments (.venv/) - - [x] Removed large model files (*.pt) - will download on DGX - - [x] Removed redundant analysis scripts - - [x] Removed documentation clutter - -- [x] Files prepared: - - [x] .gitignore created (proper exclusions) - - [x] requirements-prod.txt created - - [x] LICENSE file present - - [x] Deployment scripts created - -- [x] Git status: - - [x] All changes staged - - [x] Ready to commit - -## Deployment Steps - -### Step 1: Transfer Code to DGX -```bash -rsync -avz --exclude='.venv' --exclude='node_modules' \ - /path/to/DrivingRepo/ \ - :/tmp/sensorflow-deploy/ -``` - -### Step 2: Execute Deployment on DGX -```bash -ssh -cd /tmp/sensorflow-deploy -sudo bash deploy/setup-dgx.sh /tmp/sensorflow-deploy -``` - -### Step 3: DNS Configuration (GoDaddy) -- [ ] Add A record: sensorflow.wael.bot → [DGX IP] -- [ ] Set TTL to 3600 -- [ ] Wait for DNS propagation (5-10 minutes) - -## Post-Deployment (Verify on DGX) - -### Services -- [ ] Check sensorflow-studio service: `sudo systemctl status sensorflow-studio` -- [ ] Check accident-analysis service: `sudo systemctl status accident-analysis` -- [ ] Check nginx: `sudo systemctl status nginx` - -### API Health Checks -```bash -# From DGX or via Tailscale -curl http://sensorflow.wael.bot/ -curl http://sensorflow.wael.bot/api/studio/docs -curl http://sensorflow.wael.bot/api/analysis/docs -curl http://sensorflow.wael.bot/health/studio -curl http://sensorflow.wael.bot/health/analysis -``` - -### Logs Verification -```bash -sudo journalctl -u sensorflow-studio -n 20 -sudo journalctl -u accident-analysis -n 20 -sudo tail -20 /var/log/nginx/access.log -``` - -### Data & Models -- [ ] YOLOv8 models downloaded: `ls -lh /opt/sensorflow/studio/yolov8*.pt` -- [ ] React frontend built: `ls -la /opt/sensorflow/studio/dist/` -- [ ] Data directory present: `ls /opt/sensorflow/studio/data/` - -### Performance -```bash -# Check resource usage -nvidia-smi # GPU load -ps aux | grep sensorflow -top -u sensorflow -``` - -## Maintenance & Monitoring - -### Daily -- [ ] Monitor error logs: `sudo journalctl -u sensorflow-studio --since="1 hour ago"` -- [ ] Check disk usage: `df -h /opt/sensorflow/` -- [ ] Verify API responding: `curl http://sensorflow.wael.bot/health/studio` - -### Weekly -- [ ] Review full logs: `sudo journalctl -u sensorflow-studio -p warning` -- [ ] Check model accuracy: Review `/opt/sensorflow/studio/runs/detect/` -- [ ] Backup database (if applicable) - -### Monthly -- [ ] Review performance metrics -- [ ] Update dependencies: `pip list --outdated` -- [ ] Test disaster recovery - -## Troubleshooting Quick Reference - -| Issue | Command | -|-------|---------| -| Service won't start | `sudo journalctl -u sensorflow-studio -n 50` | -| Port conflict | `sudo lsof -i :8001` or `:8002` | -| Nginx error | `sudo nginx -t && sudo systemctl restart nginx` | -| DNS not resolving | `nslookup sensorflow.wael.bot` | -| High memory usage | `ps aux --sort=-%mem \| grep sensorflow` | -| Can't access API | Check firewall: `sudo ufw status` | -| Models not found | `bash /opt/sensorflow/studio/deploy/download-models.sh /opt/sensorflow/studio` | - -## Rollback Procedure - -If deployment fails: - -1. Stop services: - ```bash - sudo systemctl stop sensorflow-studio accident-analysis - ``` - -2. Remove deployment: - ```bash - sudo rm -rf /opt/sensorflow - ``` - -3. Re-run deployment with fixed parameters: - ```bash - sudo bash /tmp/sensorflow-deploy/deploy/setup-dgx.sh /tmp/sensorflow-deploy - ``` - -## Completion Checklist - -- [ ] DNS resolves to DGX -- [ ] Frontend accessible at sensorflow.wael.bot -- [ ] Studio API responding at /api/studio -- [ ] Analysis API responding at /api/analysis -- [ ] WebSocket connection works for telemetry -- [ ] Models downloaded and verified -- [ ] Services auto-restart on failure -- [ ] Logs are being captured -- [ ] Performance is acceptable -- [ ] Documentation updated (this file) diff --git a/hf-space/deploy/DEPLOYMENT.md b/hf-space/deploy/DEPLOYMENT.md deleted file mode 100644 index f7949c06ce9153196294caf12a172d03926c6867..0000000000000000000000000000000000000000 --- a/hf-space/deploy/DEPLOYMENT.md +++ /dev/null @@ -1,201 +0,0 @@ -# Sensorflow Deployment Guide - DGX Spark - -## Overview - -This deployment sets up two persistent FastAPI services on your DGX Spark machine via Tailscale: -- **Sensorflow Studio**: YOLOv8 training and inference service (port 8001) -- **Accident Analysis Platform**: Data analysis and WebSocket API (port 8002) -- **Frontend**: React dashboard via Vite (served at `/`) - -All services are reverse-proxied through nginx at `sensorflow.wael.bot` - -## Prerequisites - -- DGX Spark machine accessible via Tailscale -- SSH access configured (credentials in local SSH config) -- Ubuntu/Debian-based Linux on DGX -- sudo privileges on DGX -- Node.js 18+ (for frontend build) -- Python 3.9+ - -## Quick Start - -### 1. Prepare Code on Local Machine - -```bash -cd /path/to/DrivingRepo -# Cleanup already done, verify: -ls -la | grep -v "test_\|example_\|\.venv\|yolov8.*\.pt" -``` - -### 2. Transfer Code to DGX via Tailscale SSH - -```bash -# Using your SSH config -rsync -avz --exclude='.venv' --exclude='node_modules' \ - /path/to/DrivingRepo/ \ - :/tmp/sensorflow-deploy/ -``` - -Replace `` with your DGX Tailscale hostname or IP. - -### 3. Run Deployment on DGX - -```bash -# SSH into DGX -ssh - -# Run deployment script -cd /tmp/sensorflow-deploy -chmod +x deploy/setup-dgx.sh -sudo bash deploy/setup-dgx.sh /tmp/sensorflow-deploy -``` - -### 4. Configure DNS (GoDaddy) - -Add an A record pointing to your DGX Spark's external IP or Tailscale IP: -- **Domain**: sensorflow.wael.bot -- **Type**: A -- **Value**: Your DGX Spark's IP -- **TTL**: 3600 - -If using Tailscale: -- Get your Tailscale IP: `tailscale ip -4` -- Use this IP for the A record - -## Service Management - -Check service status: -```bash -sudo systemctl status sensorflow-studio accident-analysis -``` - -View logs: -```bash -# Studio backend -sudo journalctl -u sensorflow-studio -f - -# Analysis backend -sudo journalctl -u accident-analysis -f - -# Nginx -sudo tail -f /var/log/nginx/error.log -``` - -Restart services: -```bash -sudo systemctl restart sensorflow-studio accident-analysis -``` - -Stop services: -```bash -sudo systemctl stop sensorflow-studio accident-analysis -``` - -## API Endpoints - -### Studio (YOLOv8) -- `POST /api/studio/train` - Start training -- `GET /api/studio/train/logs` - Stream training logs -- `POST /api/studio/infer` - Run inference -- `POST /api/studio/export` - Export model -- Interactive docs: `/api/studio/docs` - -### Analysis (Accident Data) -- `POST /api/analysis/upload` - Upload accident data -- `GET /api/analysis/analyze` - Analyze data -- `WS /ws/telemetry` - Real-time telemetry WebSocket -- Interactive docs: `/api/analysis/docs` - -### Frontend -- `GET /` - React dashboard - -## Directory Structure on DGX - -``` -/opt/sensorflow/ -├── studio/ -│ ├── app_backend.py -│ ├── main.py -│ ├── venv/ # Python venv -│ ├── dist/ # Built React app -│ ├── src/ # React source -│ ├── requirements-prod.txt -│ └── deploy/ -├── analysis/ -│ ├── main.py -│ ├── venv/ -│ └── accident_*.py -└── models/ - ├── yolov8n.pt - └── yolov8m.pt -``` - -## Monitoring - -Check if services are responding: -```bash -curl http://sensorflow.wael.bot/health/studio -curl http://sensorflow.wael.bot/health/analysis -curl http://sensorflow.wael.bot/ -``` - -Monitor system resources: -```bash -# On DGX -nvidia-smi # GPU usage -top -u sensorflow # CPU/Memory -``` - -## Troubleshooting - -### Services fail to start -```bash -sudo journalctl -u sensorflow-studio -n 50 -sudo systemctl status sensorflow-studio -``` - -### Port conflicts -```bash -sudo lsof -i :8001 # Check port 8001 -sudo lsof -i :8002 # Check port 8002 -``` - -### Nginx issues -```bash -sudo nginx -t # Test config -sudo systemctl restart nginx -tail -f /var/log/nginx/error.log -``` - -### Model download failed -```bash -# Manual download -cd /opt/sensorflow/studio -bash deploy/download-models.sh . -``` - -## Updates & Maintenance - -To update code: -```bash -# On DGX -cd /opt/sensorflow/studio -git pull # if using git -# or rsync again from local machine - -# Restart services -sudo systemctl restart sensorflow-studio accident-analysis -``` - -## License - -This project is released under **CC BY-NC-SA 4.0**. -See LICENSE file for details. - -## Support - -For issues or questions, check: -1. Service logs: `sudo journalctl -u sensorflow-studio` -2. Nginx config: `/etc/nginx/sites-available/sensorflow.conf` -3. Python venv activation: `/opt/sensorflow/studio/venv/bin/python` diff --git a/hf-space/deploy/accident-analysis.service b/hf-space/deploy/accident-analysis.service deleted file mode 100644 index e9e69e18c59cf3fc217623998dee6ceee98ed654..0000000000000000000000000000000000000000 --- a/hf-space/deploy/accident-analysis.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Accident Analysis Platform API -After=network.target -StartLimitIntervalSec=0 - -[Service] -Type=simple -User=sensorflow -WorkingDirectory=/opt/sensorflow/analysis -ExecStart=/opt/sensorflow/analysis/venv/bin/uvicorn main:app --host 127.0.0.1 --port 8002 --workers 4 -Restart=always -RestartSec=10 -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/hf-space/deploy/download-models.sh b/hf-space/deploy/download-models.sh deleted file mode 100644 index 729851ba1793902fd318dc9c156e8ecfc0645f7b..0000000000000000000000000000000000000000 --- a/hf-space/deploy/download-models.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# Download YOLOv8 models required for deployment - -set -e - -MODEL_DIR="${1:-.}" -mkdir -p "$MODEL_DIR" - -echo "Downloading YOLOv8 models..." - -# YOLOv8 Nano (small/fast) -if [ ! -f "$MODEL_DIR/yolov8n.pt" ]; then - echo "Downloading yolov8n.pt..." - wget -q https://github.com/ultralytics/assets/releases/download/v8.1.0/yolov8n.pt -O "$MODEL_DIR/yolov8n.pt" - echo "✓ yolov8n.pt downloaded" -fi - -# YOLOv8 Medium (balanced) -if [ ! -f "$MODEL_DIR/yolov8m.pt" ]; then - echo "Downloading yolov8m.pt..." - wget -q https://github.com/ultralytics/assets/releases/download/v8.1.0/yolov8m.pt -O "$MODEL_DIR/yolov8m.pt" - echo "✓ yolov8m.pt downloaded" -fi - -echo "Models ready at $MODEL_DIR" - -# SAM ViT-B checkpoint for 3D perception pipeline -SAM_DIR="$MODEL_DIR/models" -mkdir -p "$SAM_DIR" -if [ ! -f "$SAM_DIR/sam_vit_b.pth" ]; then - echo "Downloading SAM ViT-B checkpoint..." - wget -q https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth -O "$SAM_DIR/sam_vit_b.pth" - echo "✓ sam_vit_b.pth downloaded" -fi - -echo "All models ready." diff --git a/hf-space/deploy/huggingface/README.md b/hf-space/deploy/huggingface/README.md deleted file mode 100644 index fc97ecbdf14a72fe720138f5e395d0b946a57008..0000000000000000000000000000000000000000 --- a/hf-space/deploy/huggingface/README.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: Sensorflow Studio -emoji: 🛰️ -colorFrom: indigo -colorTo: blue -sdk: docker -app_port: 7860 -pinned: false ---- - -# Sensorflow Studio - -An autonomous-driving data platform: dataset exploration, label evaluation, -MITL (model-in-the-loop) review tooling, and the SSAM Safety Dashboard for -statewide surrogate-safety analytics. - -## What's served - -- `/` — legacy static UI (dataset / training studio) -- `/dashboard/` — React (Vite) frontend, including the SSAM Safety Dashboard -- `/api/...` — FastAPI backend endpoints (e.g. `POST /api/ssam/statewide`) - -## Notes for this Space - -This is a CPU Space running a slim dependency set. Features requiring GPU or -local services — YOLO training, SAM-assisted annotation, and the Ollama -copilot — are not functional here; the dashboards and analytics APIs are. - -Deployed from the project repository via -`deploy/huggingface/deploy.py` (uploads files with `huggingface_hub`; the -frontend is built inside the Docker image). diff --git a/hf-space/deploy/huggingface/deploy.py b/hf-space/deploy/huggingface/deploy.py deleted file mode 100644 index 6f494fbf9b2821bd44adf3cd333db0cecf25bbba..0000000000000000000000000000000000000000 --- a/hf-space/deploy/huggingface/deploy.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -"""Deploy Sensorflow Studio to a Hugging Face Space (Docker SDK). - -Idempotent / re-runnable: creates the Space if missing, uploads the current -repo state (frontend is built inside the Docker image on HF's side), waits for -the build, and smoke-tests the deployed endpoints. - -Usage: - .venv/bin/python deploy/huggingface/deploy.py [--space-name NAME] [--no-wait] - -Requires a Hugging Face token with write access (`hf auth login` or HF_TOKEN). -""" -import argparse -import json -import sys -import time -import urllib.request -from pathlib import Path - -from huggingface_hub import HfApi -from huggingface_hub.errors import LocalTokenNotFoundError - -REPO_ROOT = Path(__file__).resolve().parents[2] - -# Mirrors .dockerignore, plus files that must not reach the Space repo as-is. -IGNORE_PATTERNS = [ - "node_modules/**", - ".venv/**", - ".git/**", - "runs/**", - "dist/**", - "models/**", - "*.pt", - "**/*.pt", - "**/__pycache__/**", - ".pytest_cache/**", - ".claude/**", - ".DS_Store", - "**/.DS_Store", - "README.md", # root project README; the Space README is uploaded separately -] - -TERMINAL_ERROR_STAGES = {"BUILD_ERROR", "RUNTIME_ERROR", "CONFIG_ERROR", "DELETING"} - - -def http_check(url: str, method: str = "GET", body: dict | None = None) -> str: - data = json.dumps(body).encode() if body is not None else None - req = urllib.request.Request( - url, data=data, method=method, - headers={"Content-Type": "application/json"} if body is not None else {}, - ) - try: - with urllib.request.urlopen(req, timeout=30) as resp: - snippet = resp.read(200).decode(errors="replace").replace("\n", " ") - return f"{resp.status} {snippet[:120]}" - except Exception as exc: # noqa: BLE001 - smoke test, report anything - return f"FAILED: {exc}" - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--space-name", default="sensorflow-studio") - parser.add_argument("--no-wait", action="store_true", - help="upload only; don't wait for the build") - parser.add_argument("--timeout-minutes", type=float, default=15) - args = parser.parse_args() - - api = HfApi() - try: - user = api.whoami()["name"] - except LocalTokenNotFoundError: - print("ERROR: no Hugging Face token found. Run `hf auth login` " - "(or set HF_TOKEN) with a WRITE token, then re-run this script.") - return 1 - - repo_id = f"{user}/{args.space_name}" - print(f"Deploying to Space: {repo_id}") - - api.create_repo(repo_id, repo_type="space", space_sdk="docker", exist_ok=True) - - print("Uploading repository files...") - api.upload_folder( - folder_path=str(REPO_ROOT), - repo_id=repo_id, - repo_type="space", - ignore_patterns=IGNORE_PATTERNS, - commit_message="Deploy Sensorflow Studio", - ) - # The Space README (with docker frontmatter) replaces the project README. - api.upload_file( - path_or_fileobj=str(REPO_ROOT / "deploy" / "huggingface" / "README.md"), - path_in_repo="README.md", - repo_id=repo_id, - repo_type="space", - commit_message="Update Space README", - ) - - space_url = f"https://{user}-{args.space_name.replace('_', '-')}.hf.space" - print(f"Space page: https://huggingface.co/spaces/{repo_id}") - print(f"App URL: {space_url}/") - - if args.no_wait: - return 0 - - print("Waiting for the Space build (this can take several minutes)...") - deadline = time.time() + args.timeout_minutes * 60 - stage = None - while time.time() < deadline: - runtime = api.get_space_runtime(repo_id) - if runtime.stage != stage: - stage = runtime.stage - print(f" stage: {stage}") - if stage == "RUNNING": - break - if stage in TERMINAL_ERROR_STAGES: - print(f"Build/runtime failed (stage={stage}). Check logs at " - f"https://huggingface.co/spaces/{repo_id}?logs=build") - return 2 - time.sleep(15) - else: - print(f"Timed out after {args.timeout_minutes} min (last stage: {stage}).") - return 3 - - print("Space is RUNNING. Verifying endpoints...") - print(f" GET / -> {http_check(space_url + '/')}") - print(f" GET /dashboard/ -> {http_check(space_url + '/dashboard/')}") - print(f" POST /api/ssam/statewide -> " - f"{http_check(space_url + '/api/ssam/statewide', 'POST', {'page': 1, 'page_size': 1})}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/hf-space/deploy/huggingface/requirements-space.txt b/hf-space/deploy/huggingface/requirements-space.txt deleted file mode 100644 index 40cd52b062e830b2f7bdf97b65b48aac35e328f9..0000000000000000000000000000000000000000 --- a/hf-space/deploy/huggingface/requirements-space.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Slim dependency set for the Hugging Face Space (CPU, limited build time). -# Heavy GPU/vision deps from requirements.txt (torch, ultralytics, opencv) are -# lazily imported inside route handlers and intentionally excluded here. -# Versions mirror the working local .venv where pinned in requirements.txt. -fastapi==0.140.0 -uvicorn==0.51.0 -pydantic==2.13.4 -httpx==0.28.1 -numpy==2.5.1 -scipy -pandas==3.0.5 -PyYAML==6.0.3 -python-multipart==0.0.32 -scikit-learn -requests==2.34.2 diff --git a/hf-space/deploy/nginx.conf b/hf-space/deploy/nginx.conf deleted file mode 100644 index 346ad42cad533923decd99c1b2ffababe9ef0395..0000000000000000000000000000000000000000 --- a/hf-space/deploy/nginx.conf +++ /dev/null @@ -1,69 +0,0 @@ -upstream studio_backend { - server 127.0.0.1:8001; -} - -upstream analysis_backend { - server 127.0.0.1:8002; -} - -server { - listen 80; - server_name sensorflow.wael.bot; - client_max_body_size 100M; - - # Redirect HTTP to HTTPS (when cert is ready) - # return 301 https://$server_name$request_uri; - - # Sensorflow Studio API - location /api/studio/ { - proxy_pass http://studio_backend/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 300s; - proxy_connect_timeout 300s; - } - - # Accident Analysis API - location /api/analysis/ { - proxy_pass http://analysis_backend/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 300s; - proxy_connect_timeout 300s; - } - - # WebSocket support for Accident Analysis - location /ws/ { - proxy_pass http://analysis_backend/ws/; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_read_timeout 86400; - } - - # React Frontend - location / { - root /opt/sensorflow/studio/dist; - try_files $uri $uri/ /index.html; - expires 1h; - add_header Cache-Control "public, max-age=3600"; - } - - # Health checks - location /health/studio { - proxy_pass http://studio_backend/docs; - access_log off; - } - - location /health/analysis { - proxy_pass http://analysis_backend/docs; - access_log off; - } -} diff --git a/hf-space/deploy/sensorflow-studio.service b/hf-space/deploy/sensorflow-studio.service deleted file mode 100644 index 6a21a731e96820360241caa68d1aca11aebd613d..0000000000000000000000000000000000000000 --- a/hf-space/deploy/sensorflow-studio.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Sensorflow Studio Backend (YOLOv8 Training/Inference) -After=network.target -StartLimitIntervalSec=0 - -[Service] -Type=simple -User=sensorflow -WorkingDirectory=/opt/sensorflow/studio -ExecStart=/opt/sensorflow/studio/venv/bin/uvicorn app_backend:app --host 127.0.0.1 --port 8001 --workers 4 -Restart=always -RestartSec=10 -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/hf-space/deploy/setup-dgx.sh b/hf-space/deploy/setup-dgx.sh deleted file mode 100644 index 05f30c7e28acfe9f077870fdbfb5f914a57b1971..0000000000000000000000000000000000000000 --- a/hf-space/deploy/setup-dgx.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -# Sensorflow DGX Spark Deployment Setup -# Run this on the DGX Spark machine - -set -e - -APP_BASE="/opt/sensorflow" -STUDIO_DIR="$APP_BASE/studio" -ANALYSIS_DIR="$APP_BASE/analysis" -REPO_URL="${1:-$(cd /tmp && pwd)}" - -echo "🚀 Sensorflow Deployment to DGX Spark" -echo "=======================================" -echo "Base directory: $APP_BASE" -echo "Repo source: $REPO_URL" - -# Create application user if needed -if ! id "sensorflow" &>/dev/null; then - echo "Creating sensorflow system user..." - sudo useradd -r -s /bin/bash -d $APP_BASE sensorflow -fi - -# Setup directories -echo "Setting up directories..." -sudo mkdir -p $STUDIO_DIR $ANALYSIS_DIR -sudo chown -R sensorflow:sensorflow $APP_BASE - -# Copy code -echo "Copying application code..." -sudo cp -r "$REPO_URL"/* $STUDIO_DIR/ 2>/dev/null || true -mkdir -p $ANALYSIS_DIR -sudo cp $STUDIO_DIR/main.py $ANALYSIS_DIR/ - -# Create Python virtual environments -echo "Creating virtual environments..." -sudo -u sensorflow python3 -m venv $STUDIO_DIR/venv -sudo -u sensorflow python3 -m venv $ANALYSIS_DIR/venv - -# Install dependencies -echo "Installing Python dependencies..." -sudo -u sensorflow $STUDIO_DIR/venv/bin/pip install --upgrade pip setuptools wheel -sudo -u sensorflow $STUDIO_DIR/venv/bin/pip install -r $STUDIO_DIR/requirements-prod.txt -sudo -u sensorflow $ANALYSIS_DIR/venv/bin/pip install -r $STUDIO_DIR/requirements-prod.txt - -# Download models -echo "Downloading YOLOv8 models..." -sudo -u sensorflow bash $STUDIO_DIR/deploy/download-models.sh $STUDIO_DIR - -# Build React frontend -echo "Building React frontend..." -cd $STUDIO_DIR -if [ -f "package.json" ]; then - npm install - npm run build - echo "Frontend built at $STUDIO_DIR/dist" -fi - -# Setup systemd services -echo "Installing systemd services..." -sudo cp $STUDIO_DIR/deploy/sensorflow-studio.service /etc/systemd/system/ -sudo cp $STUDIO_DIR/deploy/accident-analysis.service /etc/systemd/system/ -sudo systemctl daemon-reload - -# Setup nginx -echo "Configuring nginx..." -sudo cp $STUDIO_DIR/deploy/nginx.conf /etc/nginx/sites-available/sensorflow.conf -sudo ln -sf /etc/nginx/sites-available/sensorflow.conf /etc/nginx/sites-enabled/ 2>/dev/null || true -sudo nginx -t -sudo systemctl restart nginx - -# Start services -echo "Starting services..." -sudo systemctl enable sensorflow-studio accident-analysis -sudo systemctl start sensorflow-studio accident-analysis -sudo systemctl status sensorflow-studio accident-analysis - -echo "" -echo "✅ Deployment Complete!" -echo "=======================================" -echo "Services:" -echo " - Sensorflow Studio: http://sensorflow.wael.bot/api/studio" -echo " - Accident Analysis: http://sensorflow.wael.bot/api/analysis" -echo " - Frontend: http://sensorflow.wael.bot" -echo "" -echo "Monitor logs with:" -echo " sudo journalctl -u sensorflow-studio -f" -echo " sudo journalctl -u accident-analysis -f" diff --git a/hf-space/index.html b/hf-space/index.html deleted file mode 100644 index 7d62cb4a4ce5137ead520dfa50d967cff5d7441b..0000000000000000000000000000000000000000 --- a/hf-space/index.html +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - Sensorflow Studio — L4 Perception Label Evaluation - - -
    - diff --git a/hf-space/infer.py b/hf-space/infer.py deleted file mode 100644 index 5b43f4b98b0d9405f8291cd42d59b8df642a54a1..0000000000000000000000000000000000000000 --- a/hf-space/infer.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 -""" -Inference script for YOLO Auto-Labeler. -Runs predictions on images and outputs structured JSON labels + annotated images. -Usage: python infer.py --source data/test.jpg --weights runs/detect/coco_finetuned/weights/best.pt -""" - -import argparse -import json -from pathlib import Path -from ultralytics import YOLO -from taxonomy import TAXONOMY_MAP - -def main(): - parser = argparse.ArgumentParser(description="Run YOLOv8 inference and save structured predictions") - parser.add_argument("--source", type=str, required=True, help="Path to image file or directory") - parser.add_argument("--weights", type=str, default="runs/detect/coco_finetuned/weights/best.pt", help="Path to model weights") - parser.add_argument("--conf", type=float, default=0.25, help="Confidence threshold") - parser.add_argument("--iou", type=float, default=0.45, help="NMS IoU threshold") - parser.add_argument("--device", default="0", help="GPU device ID, or cpu/-1 for CPU") - parser.add_argument("--output", type=str, default="runs/infer", help="Directory to save predictions and images") - - args = parser.parse_args() - - # Resolve device parameter - if args.device == "dgx-spark": - print("⚡ [DGX-Spark] Connected to Tailscale MagicDNS endpoint (dgx-spark.tail16d8d9.ts.net)") - print("⚡ [DGX-Spark] Offloading inference workload to NVIDIA DGX Spark remote cluster.") - device = "0" - else: - try: - device = int(args.device) - if device < 0: - device = "cpu" - except ValueError: - device = args.device - - print(f"Loading model: {args.weights}") - model = YOLO(args.weights) - - # Create output directory - output_dir = Path(args.output) - output_dir.mkdir(parents=True, exist_ok=True) - - # Identify source images - source_path = Path(args.source) - if source_path.is_dir(): - image_files = [] - for ext in ["*.jpg", "*.jpeg", "*.png", "*.bmp"]: - image_files.extend(source_path.glob(ext)) - image_files.extend(source_path.glob(ext.upper())) - image_files = sorted(list(set(image_files))) - else: - image_files = [source_path] - - if not image_files: - print(f"No images found at source: {args.source}") - return - - print(f"Running inference on {len(image_files)} image(s)...") - - predictions = [] - - for img_path in image_files: - print(f" Predicting: {img_path.name}") - results = model.predict( - source=str(img_path), - conf=args.conf, - iou=args.iou, - device=device, - verbose=False - ) - - # Save annotated image - result = results[0] - annotated_path = output_dir / f"annotated_{img_path.name}" - result.save(filename=str(annotated_path)) - - # Extract structured predictions - for box in result.boxes: - class_id = int(box.cls[0]) - class_name = model.names[class_id] - confidence = float(box.conf[0]) - bbox_coords = box.xyxy[0].tolist() # [x1, y1, x2, y2] - - pred = { - "image": img_path.name, - "class_id": class_id, - "class_name": class_name, - "taxonomy": TAXONOMY_MAP.get(class_name, "Unknown"), - "confidence": confidence, - "bbox": { - "x1": bbox_coords[0], - "y1": bbox_coords[1], - "x2": bbox_coords[2], - "y2": bbox_coords[3], - }, - } - predictions.append(pred) - - # Save predictions as JSON - predictions_file = output_dir / "predictions.json" - with open(predictions_file, "w") as f: - json.dump(predictions, f, indent=2) - - print(f"✓ Inference complete! Processed {len(image_files)} images.") - print(f"✓ Predictions saved to: {predictions_file}") - print(f"✓ Annotated images saved to: {output_dir}/") - -if __name__ == "__main__": - main() diff --git a/hf-space/main.py b/hf-space/main.py deleted file mode 100644 index d23c152568005097a8c38357713eb75c0f0f0d23..0000000000000000000000000000000000000000 --- a/hf-space/main.py +++ /dev/null @@ -1,386 +0,0 @@ -""" -Accident Analysis API Gateway - Simplified Version -For demonstration purposes, providing core functionality only. -""" - -import io -import logging -from typing import Dict, Any, List, Optional -from fastapi import FastAPI, UploadFile, File, HTTPException, WebSocket, WebSocketDisconnect -from fastapi.responses import HTMLResponse, JSONResponse -from fastapi.middleware.cors import CORSMiddleware -import pandas as pd -import asyncio -from contextlib import asynccontextmanager - -# Simplified implementations to avoid dependency issues -class MockAccidentValidator: - def __init__(self, df): - self.df = df - - def validate_schema(self): - class Report: - def __init__(self): - self.valid = True - self.errors = [] - self.quality_score = 0.95 - return Report() - -class MockAccidentDataCleaner: - def __init__(self, df): - self.df = df - - def clean_dataset(self): - # Remove any rows with missing values - return self.df.dropna() - -class MockAccidentAnalysisEngine: - def __init__(self, df): - self.df = df - - def run_full_pipeline(self): - return { - "metrics": { - "total_records": len(self.df), - "geospatial": { - "hotspots": [ - {"lat": 37.7749, "lng": -122.4194, "count": 25}, - {"lat": 34.0522, "lng": -118.2437, "count": 18} - ] - } - }, - "summary": "Analysis complete for dataset" - } - -class MockAccidentInsights: - def __init__(self, metrics): - self.metrics = metrics - - def get_risk_profile(self): - return { - "risk_index": 0.75, - "high_risk_factors": ["urban", "night_hours"], - "recommendations": [ - "Increase street lighting in high-risk areas", - "Implement additional safety measures near hotspots" - ], - "confidence_score": 0.89 - } - -class MockAccidentReport: - def __init__(self, metrics, insights): - self.metrics = metrics - self.insights = insights - - def export_to_html(self): - return f""" - - Executive Report - -

    Accident Analysis Executive Report

    -

    Total Records Processed: {self.metrics['metrics']['total_records']}

    -

    Risk Index: {self.insights.get('risk_index', 'N/A')}

    -

    Key Findings

    -
      -
    • High-risk factors identified
      • -
    • Hotspot locations provided
      • -
    - - - """ - -# Logging configuration -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') -logger = logging.getLogger(__name__) - -# Lifecycle management for the application -@asynccontextmanager -async def lifespan(app: FastAPI): - logger.info("Application starting up") - yield - logger.info("Application shutting down") - -app = FastAPI( - title="Accident Analysis Platform", - description="Simplified API Engine for demonstration purposes", - version="1.0.0", - lifespan=lifespan -) - -# Add CORS middleware to handle cross-origin requests -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -# Thread-safe cache with expiration for processed datasets -class EngineCache: - """Thread-safe cache implementation with TTL for processed datasets.""" - - def __init__(self): - self._cache: Dict[str, Dict] = {} - self._lock = asyncio.Lock() - - async def set(self, key: str, value: Dict[str, Any], ttl: int = 3600): - """Set a cached value with TTL.""" - async with self._lock: - self._cache[key] = { - "data": value, - "timestamp": asyncio.get_event_loop().time(), - "ttl": ttl - } - - async def get(self, key: str) -> Optional[Dict[str, Any]]: - """Get a cached value, checking TTL.""" - async with self._lock: - if key not in self._cache: - return None - - item = self._cache[key] - current_time = asyncio.get_event_loop().time() - - # Check if expired - if current_time - item["timestamp"] > item["ttl"]: - del self._cache[key] - return None - - return item["data"] - - async def cleanup_expired(self): - """Remove expired cache entries.""" - async with self._lock: - current_time = asyncio.get_event_loop().time() - expired_keys = [ - key for key, item in self._cache.items() - if current_time - item["timestamp"] > item["ttl"] - ] - for key in expired_keys: - del self._cache[key] - -# Global cache instance -engine_cache = EngineCache() - -class ConnectionManager: - """Manages WebSocket clients subscribing to real-time telemetry updates.""" - - def __init__(self): - self.active_connections: List[WebSocket] = [] - self._lock = asyncio.Lock() - - async def connect(self, websocket: WebSocket): - """Add a new WebSocket connection.""" - await websocket.accept() - async with self._lock: - self.active_connections.append(websocket) - - def disconnect(self, websocket: WebSocket): - """Remove a disconnected WebSocket connection.""" - try: - with self._lock: - self.active_connections.remove(websocket) - except ValueError: - pass # Already disconnected - - async def broadcast(self, message: str): - """Broadcast a message to all active connections.""" - async with self._lock: - # Create list of tasks to avoid potential modification during iteration - tasks = [] - for connection in self.active_connections.copy(): - try: - tasks.append(connection.send_text(message)) - except Exception as e: - logger.warning(f"Failed to send to WebSocket: {e}") - # Connection might be closed, remove it - try: - self.active_connections.remove(connection) - except ValueError: - pass # Already removed - - # Execute all sends concurrently - await asyncio.gather(*tasks, return_exceptions=True) - -manager = ConnectionManager() - -@app.post("/api/analyze") -async def analyze_accident_data(file: UploadFile = File(...)): - """ - Uploads a SWITRS formatted CSV dataset, runs validation, cleaning, and initiates the engine. - - Returns: - JSON response with analysis results - """ - logger.info(f"Received file upload for analysis: {file.filename}") - - if not file.filename or not file.filename.endswith(".csv"): - raise HTTPException(status_code=400, detail="Only CSV datasets are accepted.") - - try: - # Validate file size (limit to 50MB) - content = await file.read() - if len(content) > 50 * 1024 * 1024: # 50 MB limit - raise HTTPException(status_code=413, detail="File too large. Maximum size is 50MB.") - - # Parse CSV with error handling - df = pd.read_csv(io.BytesIO(content)) - logger.info(f"CSV loaded successfully, shape: {df.shape}") - - # Mock Validation - validator = MockAccidentValidator(df) - report = validator.validate_schema() - if not report.valid: - logger.warning(f"Dataset validation failed: {report.errors}") - - # Mock Cleaning - cleaner = MockAccidentDataCleaner(df) - clean_df = cleaner.clean_dataset() - - if clean_df.empty: - raise HTTPException(status_code=422, detail="Cleaning produced an empty dataset.") - - # Mock Analysis - engine = MockAccidentAnalysisEngine(clean_df) - metrics = engine.run_full_pipeline() - - # Mock Insights - insights = MockAccidentInsights(metrics) - profile = insights.get_risk_profile() - - # Cache for sub-resource queries with TTL (1 hour) - await engine_cache.set("latest", { - "metrics": metrics, - "insights": profile, - "processing_time": len(clean_df) # for reference - }, ttl=3600) - - # Broadcast trigger over WebSockets - broadcast_message = f"New dataset processed. Total records: {len(clean_df)}. Risk index: {profile.get('risk_index', 'N/A')}" - await manager.broadcast(broadcast_message) - - return { - "status": "success", - "records_processed": len(clean_df), - "quality_score": report.quality_score, - "insights": profile, - "processing_time_seconds": len(clean_df) # Add processing info - } - - except pd.errors.EmptyDataError: - raise HTTPException(status_code=400, detail="Empty CSV file provided.") - except pd.errors.ParserError as e: - raise HTTPException(status_code=400, detail=f"Invalid CSV format: {str(e)}") - except Exception as e: - logger.error(f"Error in dataset ingestion pipeline: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail="Pipeline error occurred during processing") - -@app.get("/api/hotspots") -async def get_hotspots(): - """ - Retrieves geographic hotspots from the latest processed run. - - Returns: - JSON response with geospatial hotspots - """ - try: - cached_data = await engine_cache.get("latest") - if not cached_data: - raise HTTPException(status_code=404, detail="No processed metrics found. Please POST to /api/analyze first.") - - geo = cached_data["metrics"].get("geospatial", {}) - hotspots = geo.get("hotspots", []) - - return { - "status": "success", - "hotspots": hotspots, - "count": len(hotspots) - } - except Exception as e: - logger.error(f"Error retrieving hotspots: {str(e)}") - raise HTTPException(status_code=500, detail="Failed to retrieve hotspots") - -@app.get("/api/insights") -async def get_insights(): - """ - Retrieves high-level risk factors and recommendations. - - Returns: - JSON response with risk insights - """ - try: - cached_data = await engine_cache.get("latest") - if not cached_data: - raise HTTPException(status_code=404, detail="No processed metrics found. Please POST to /api/analyze first.") - - insights = cached_data["insights"] - return { - "status": "success", - "insights": insights - } - except Exception as e: - logger.error(f"Error retrieving insights: {str(e)}") - raise HTTPException(status_code=500, detail="Failed to retrieve insights") - -@app.get("/api/report", response_class=HTMLResponse) -async def get_executive_report(): - """ - Generates and downloads the executive HTML summary report. - - Returns: - HTML content of the generated report - """ - try: - cached_data = await engine_cache.get("latest") - if not cached_data: - return HTMLResponse(content="

    No report generated. Please analyze a dataset first.

    ", status_code=404) - - report = MockAccidentReport( - metrics=cached_data["metrics"], - insights=cached_data["insights"] - ) - html_content = report.export_to_html() - return HTMLResponse(content=html_content, status_code=200) - - except Exception as e: - logger.error(f"Error generating report: {str(e)}") - raise HTTPException(status_code=500, detail="Failed to generate report") - -@app.websocket("/ws/stream") -async def websocket_endpoint(websocket: WebSocket): - """ - Subscribes to live broadcasts of analytical status changes. - - Returns: - WebSocket connection for real-time updates - """ - await manager.connect(websocket) - try: - while True: - # Keep connection alive, listen for client messages - data = await websocket.receive_text() - await websocket.send_text(f"Telemetry heartbeat: received {len(data)} bytes") - except WebSocketDisconnect: - logger.info("WebSocket disconnected") - manager.disconnect(websocket) - except Exception as e: - logger.error(f"WebSocket error: {str(e)}") - manager.disconnect(websocket) - -@app.get("/health") -async def health_check(): - """ - Health check endpoint for monitoring. - - Returns: - JSON response indicating service status - """ - return JSONResponse( - content={ - "status": "healthy", - "service": "AccidentAnalysisAPI", - "version": "1.0.0" - }, - status_code=200 - ) diff --git a/hf-space/package.json b/hf-space/package.json deleted file mode 100644 index c13e129674430b5cb608d5f98053e64f8c048835..0000000000000000000000000000000000000000 --- a/hf-space/package.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "name": "driving-repo", - "version": "0.1.0", - "private": true, - "type": "module", - "scripts": { - "dev": "vite", - "build": "vite build", - "preview": "vite preview", - "typecheck": "tsc --noEmit" - }, - "dependencies": { - "@deck.gl/core": "^8.9.30", - "@deck.gl/geo-layers": "^8.9.30", - "@deck.gl/layers": "^8.9.30", - "@deck.gl/react": "^8.9.30", - "@emotion/react": "^11.13.0", - "@emotion/styled": "^11.13.0", - "@mui/material": "^5.16.0", - "lucide-react": "^0.453.0", - "react": "^18.3.0", - "react-dom": "^18.3.0", - "react-map-gl": "^7.1.6" - }, - "devDependencies": { - "@types/node": "^22.5.0", - "@types/react": "^18.3.3", - "@types/react-dom": "^18.3.2", - "@vitejs/plugin-react": "^4.5.0", - "typescript": "^7.0.2", - "vite": "^5.4.0" - } -} diff --git a/hf-space/requirements-3d.txt b/hf-space/requirements-3d.txt deleted file mode 100644 index 3396759f422e8d95675a89458d8439cdb23c14c8..0000000000000000000000000000000000000000 --- a/hf-space/requirements-3d.txt +++ /dev/null @@ -1,13 +0,0 @@ -# 3D Perception Pipeline dependencies (install alongside requirements.txt) --r requirements.txt - -scipy>=1.11 -filterpy>=1.4.5 -segment-anything>=1.0 -plotly>=5.0 - -# Optional: Waymo Open Dataset (heavy, install separately if needed) -# waymo-open-dataset - -# Optional: point cloud utilities -# open3d>=0.18 diff --git a/hf-space/requirements-prod.txt b/hf-space/requirements-prod.txt deleted file mode 100644 index 68515cbddb0a105dc10cd6c9f33c585a833b88a8..0000000000000000000000000000000000000000 --- a/hf-space/requirements-prod.txt +++ /dev/null @@ -1,18 +0,0 @@ -fastapi==0.140.0 -uvicorn==0.51.0 -starlette==1.3.1 -pydantic==2.13.4 -pandas==3.0.5 -polars==1.43.0 -numpy==2.5.1 -opencv-python==5.0.0.93 -ultralytics==8.4.106 -torch==2.13.0 -torchvision==0.28.0 -pillow==12.3.0 -matplotlib==3.11.1 -PyYAML==6.0.3 -requests==2.34.2 -httpx==0.28.1 -nvidia-ml-py==13.610.43 -psutil==7.2.2 diff --git a/hf-space/requirements.txt b/hf-space/requirements.txt deleted file mode 100644 index e9fffee5e673eac3b59364c4252e41bf4158772d..0000000000000000000000000000000000000000 --- a/hf-space/requirements.txt +++ /dev/null @@ -1,59 +0,0 @@ -annotated-doc==0.0.4 -annotated-types==0.8.0 -anyio==4.14.2 -certifi==2026.7.22 -charset-normalizer==3.4.9 -click==8.4.2 -contourpy==1.3.3 -cycler==0.12.1 -et_xmlfile==2.0.0 -fastapi==0.140.0 -filelock==3.32.0 -fonttools==4.63.0 -fsspec==2026.6.0 -h11==0.16.0 -hf-xet==1.5.2 -httpcore==1.0.9 -httpx==0.28.1 -huggingface_hub==1.24.0 -idna==3.18 -iniconfig==2.3.0 -Jinja2==3.1.6 -kiwisolver==1.5.0 -MarkupSafe==3.0.3 -matplotlib==3.11.1 -mpmath==1.3.0 -networkx==3.6.1 -numpy==2.5.1 -nvidia-ml-py==13.610.43 -opencv-python==5.0.0.93 -openpyxl==3.1.5 -packaging==26.2 -pandas==3.0.5 -pillow==12.3.0 -pluggy==1.6.0 -polars==1.43.0 -polars-runtime-32==1.43.0 -psutil==7.2.2 -pydantic==2.13.4 -pydantic_core==2.46.4 -Pygments==2.20.0 -pyparsing==3.3.2 -pytest==9.1.1 -python-dateutil==2.9.0.post0 -python-multipart==0.0.32 -PyYAML==6.0.3 -requests==2.34.2 -setuptools==83.0.0 -six==1.17.0 -starlette==1.3.1 -sympy==1.14.0 -torch==2.13.0 -torchvision==0.28.0 -tqdm==4.69.1 -typing-inspection==0.4.2 -typing_extensions==4.16.0 -ultralytics==8.4.106 -ultralytics-thop==2.0.20 -urllib3==2.7.0 -uvicorn==0.51.0 diff --git a/hf-space/sensorflow/__init__.py b/hf-space/sensorflow/__init__.py deleted file mode 100644 index 463b679e09fbe4c06060c42ac76642cb56bbfcf9..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Sensorflow Studio 3D perception pipeline package.""" - -from sensorflow.schemas.unified_frame import ( - UnifiedSequence, - FusedFrame, - Object3D, - GroundTruthObject, - CameraView, - LidarData, - EgoPose, - Calibration, -) -from sensorflow.schemas.taxonomy_axes import TaxonomyAxes, assign_taxonomy_axes - -__all__ = [ - "UnifiedSequence", - "FusedFrame", - "Object3D", - "GroundTruthObject", - "CameraView", - "LidarData", - "EgoPose", - "Calibration", - "TaxonomyAxes", - "assign_taxonomy_axes", -] diff --git a/hf-space/sensorflow/adapters/__init__.py b/hf-space/sensorflow/adapters/__init__.py deleted file mode 100644 index d0f60ad91621c338769b708fb66c54b22d3db825..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/adapters/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from sensorflow.adapters.alpamayo_adapter import AlpamayoAdapter -from sensorflow.adapters.waymo_adapter import WaymoAdapter - -__all__ = ["AlpamayoAdapter", "WaymoAdapter"] diff --git a/hf-space/sensorflow/adapters/alpamayo_adapter.py b/hf-space/sensorflow/adapters/alpamayo_adapter.py deleted file mode 100644 index b45a459f549fc93f6c97710dbf742c637bd1282f..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/adapters/alpamayo_adapter.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Normalize Alpamayo / PhysicalAI samples into UnifiedSequence.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, Dict, List - -import numpy as np - -from sensorflow.adapters.base import VendorAdapter -from sensorflow.schemas.taxonomy_axes import assign_taxonomy_axes -from sensorflow.schemas.unified_frame import ( - CameraView, - EgoPose, - FusedFrame, - GroundTruthObject, - LidarData, - UnifiedSequence, -) - - -def _synthetic_lidar_path(output_dir: Path, frame_id: str, annotations: List[Dict]) -> tuple[str, int]: - """Generate a minimal synthetic LiDAR point cloud for local dev.""" - output_dir.mkdir(parents=True, exist_ok=True) - lidar_path = output_dir / f"{frame_id}_lidar.bin" - points = [] - for i, ann in enumerate(annotations): - box = ann.get("box", [100, 100, 50, 50]) - cx = box[0] + box[2] / 2 - cy = box[1] + box[3] / 2 - x = (cx - 320) * 0.05 - y = (240 - cy) * 0.05 - z = 0.5 + i * 0.1 - for _ in range(50): - noise = np.random.randn(3) * 0.1 - points.append([x + noise[0], y + noise[1], z + noise[2]]) - if not points: - points = np.random.randn(100, 3).astype(np.float32) * 2 - else: - points = np.array(points, dtype=np.float32) - points.tofile(lidar_path) - return str(lidar_path), len(points) - - -def _box2d_to_bbox3d(box: List[float], conf: float) -> List[float]: - """Convert 2D box [x,y,w,h] to stub 3D bbox [x,y,z,l,w,h,yaw].""" - cx = box[0] + box[2] / 2 - cy = box[1] + box[3] / 2 - x = (cx - 320) * 0.05 - y = (240 - cy) * 0.05 - l = max(box[2] * 0.05, 1.0) - w = max(box[3] * 0.05, 0.5) - return [x, y, 0.5, l, w, 1.5, 0.0] - - -DEFAULT_ALPAMAYO_SAMPLES = { - "physical_ai": { - "dataset": "nvidia/PhysicalAI-Autonomous-Vehicles", - "frame_index": 4209, - "views": { - "front": "https://images.unsplash.com/photo-1506015391300-4802dc74de2e?w=800", - "left": "https://images.unsplash.com/photo-1519074002996-a69e7ac46a42?w=800", - "right": "https://images.unsplash.com/photo-1549317661-bd32c8ce0db2?w=800", - }, - "telemetry": {"lat": 37.774929, "lon": -122.419416, "speed_kmh": 35.8}, - "annotations": [ - {"id": 1, "label": "pedestrian", "box": [120, 240, 60, 120], "conf": 0.89}, - {"id": 2, "label": "car", "box": [340, 220, 150, 100], "conf": 0.95}, - {"id": 3, "label": "traffic_light", "box": [500, 80, 30, 60], "conf": 0.92}, - ], - }, - "nurec": { - "dataset": "nvidia/PhysicalAI-Autonomous-Vehicles-NuRec", - "frame_index": 781, - "views": { - "front": "https://images.unsplash.com/photo-1568605117036-5fe5e7bab0b7?w=800", - }, - "telemetry": {"lat": 37.783312, "lon": -122.416733, "speed_kmh": 48.2}, - "annotations": [ - {"id": 1, "label": "car", "box": [300, 180, 180, 140], "conf": 0.94}, - {"id": 2, "label": "truck", "box": [50, 150, 220, 180], "conf": 0.81}, - ], - }, -} - - -class AlpamayoAdapter(VendorAdapter): - def load(self, source: Dict[str, Any], sequence_id: str) -> UnifiedSequence: - data = source if source else DEFAULT_ALPAMAYO_SAMPLES["physical_ai"] - output_dir = Path("runs/pipeline") / sequence_id / "lidar" - speed = data.get("telemetry", {}).get("speed_kmh", 35.0) - annotations = data.get("annotations", []) - - frames: List[FusedFrame] = [] - num_frames = max(3, len(annotations)) - for i in range(num_frames): - frame_id = f"frame_{i:04d}" - lidar_path, num_points = _synthetic_lidar_path(output_dir, frame_id, annotations) - cameras = { - name: CameraView(image_path=url) - for name, url in data.get("views", {}).items() - } - gt = [] - for ann in annotations: - axes = assign_taxonomy_axes(ann["label"], speed_kmh=speed) - gt.append(GroundTruthObject( - instance_id=f"gt_{ann['id']}", - class_name=ann["label"], - confidence=ann.get("conf", 1.0), - bbox_3d=_box2d_to_bbox3d(ann["box"], ann.get("conf", 1.0)), - taxonomy_axes=axes, - )) - frames.append(FusedFrame( - frame_id=frame_id, - timestamp_us=i * 100_000, - lidar=LidarData(path=lidar_path, num_points=num_points), - cameras=cameras, - ego_pose=EgoPose(speed_kmh=speed, x=i * 1.1, y=i * 5.2), - ground_truth=gt, - )) - - return UnifiedSequence( - sequence_id=sequence_id, - vendor="alpamayo", - frames=frames, - taxonomy_manifest={"source_dataset": data.get("dataset", "alpamayo")}, - ) - - @classmethod - def load_from_file(cls, path: Path, sequence_id: str) -> UnifiedSequence: - with open(path) as f: - return cls().load(json.load(f), sequence_id) diff --git a/hf-space/sensorflow/adapters/base.py b/hf-space/sensorflow/adapters/base.py deleted file mode 100644 index 3c92781dda1ad9c99b2305a6c909f652339036e8..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/adapters/base.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Adapter protocol for vendor-specific data normalization.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Any, Dict - -from sensorflow.schemas.unified_frame import UnifiedSequence - - -class VendorAdapter(ABC): - @abstractmethod - def load(self, source: Dict[str, Any], sequence_id: str) -> UnifiedSequence: - """Convert vendor-specific data into a UnifiedSequence.""" diff --git a/hf-space/sensorflow/adapters/waymo_adapter.py b/hf-space/sensorflow/adapters/waymo_adapter.py deleted file mode 100644 index b4edcfbc400d730b0b4f99d6a3921b57b5533eea..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/adapters/waymo_adapter.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Normalize Waymo data into UnifiedSequence.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, Dict, List - -import numpy as np - -from sensorflow.adapters.base import VendorAdapter -from sensorflow.schemas.taxonomy_axes import assign_taxonomy_axes -from sensorflow.schemas.unified_frame import ( - CameraView, - EgoPose, - FusedFrame, - GroundTruthObject, - LidarData, - UnifiedSequence, -) - -WAYMO_TYPE_MAP = { - 1: "vehicle", - 2: "pedestrian", - 3: "sign", - 4: "cyclist", -} - - -def _default_waymo_shard() -> List[Dict[str, Any]]: - """Built-in shard when Waymo SDK and external files are unavailable.""" - return [ - { - "frame_id": "waymo_0000", - "timestamp_us": 0, - "lidar_points": 2048, - "ego_speed_kmh": 45.0, - "labels": [ - {"instance_id": "wgt_1", "type": 1, "center_x": 10.0, "center_y": 2.0, - "center_z": 0.5, "length": 4.5, "width": 1.8, "height": 1.5, "heading": 0.1}, - {"instance_id": "wgt_2", "type": 2, "center_x": 15.0, "center_y": -1.5, - "center_z": 0.9, "length": 0.5, "width": 0.5, "height": 1.7, "heading": 1.2}, - ], - }, - { - "frame_id": "waymo_0001", - "timestamp_us": 100_000, - "lidar_points": 2100, - "ego_speed_kmh": 45.5, - "labels": [ - {"instance_id": "wgt_1", "type": 1, "center_x": 10.5, "center_y": 2.1, - "center_z": 0.5, "length": 4.5, "width": 1.8, "height": 1.5, "heading": 0.1}, - {"instance_id": "wgt_2", "type": 2, "center_x": 15.2, "center_y": -1.4, - "center_z": 0.9, "length": 0.5, "width": 0.5, "height": 1.7, "heading": 1.1}, - ], - }, - { - "frame_id": "waymo_0002", - "timestamp_us": 200_000, - "lidar_points": 1980, - "ego_speed_kmh": 46.0, - "labels": [ - {"instance_id": "wgt_1", "type": 1, "center_x": 11.0, "center_y": 2.0, - "center_z": 0.5, "length": 4.5, "width": 1.8, "height": 1.5, "heading": 0.0}, - ], - }, - ] - - -def _write_synthetic_lidar(path: Path, num_points: int, labels: List[Dict]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - points = [] - for lbl in labels: - cx, cy, cz = lbl["center_x"], lbl["center_y"], lbl["center_z"] - for _ in range(max(20, num_points // max(len(labels), 1) // 10)): - points.append([cx + np.random.randn() * 0.2, cy + np.random.randn() * 0.2, cz + np.random.randn() * 0.1]) - if not points: - points = np.random.randn(num_points, 3).astype(np.float32) - else: - points = np.array(points, dtype=np.float32) - points.tofile(path) - - -class WaymoAdapter(VendorAdapter): - def load(self, source: Dict[str, Any], sequence_id: str) -> UnifiedSequence: - shard_dir = Path("runs/pipeline/waymo_shards") - shard_path = shard_dir / f"{source.get('shard_id', 'default')}.json" - - if shard_path.exists(): - with open(shard_path) as f: - shards = json.load(f) - elif source.get("frames"): - shards = source["frames"] - else: - shards = _default_waymo_shard() - shard_dir.mkdir(parents=True, exist_ok=True) - with open(shard_path, "w") as f: - json.dump(shards, f, indent=2) - - frames: List[FusedFrame] = [] - lidar_dir = Path("runs/pipeline") / sequence_id / "lidar" - - for shard in shards: - frame_id = shard["frame_id"] - lidar_path = lidar_dir / f"{frame_id}_lidar.bin" - labels = shard.get("labels", []) - num_points = shard.get("lidar_points", 2048) - _write_synthetic_lidar(lidar_path, num_points, labels) - - gt = [] - speed = shard.get("ego_speed_kmh", 40.0) - for lbl in labels: - class_name = WAYMO_TYPE_MAP.get(lbl.get("type", 1), "vehicle") - axes = assign_taxonomy_axes(class_name, speed_kmh=speed) - gt.append(GroundTruthObject( - instance_id=lbl["instance_id"], - class_name=class_name, - confidence=1.0, - bbox_3d=[ - lbl["center_x"], lbl["center_y"], lbl["center_z"], - lbl["length"], lbl["width"], lbl["height"], lbl["heading"], - ], - taxonomy_axes=axes, - )) - - frames.append(FusedFrame( - frame_id=frame_id, - timestamp_us=shard.get("timestamp_us", 0), - lidar=LidarData(path=str(lidar_path), num_points=num_points), - cameras={"front": CameraView(image_path=shard.get("image_path", ""))}, - ego_pose=EgoPose(speed_kmh=speed), - ground_truth=gt, - )) - - return UnifiedSequence( - sequence_id=sequence_id, - vendor="waymo", - frames=frames, - taxonomy_manifest={"shard": str(shard_path)}, - ) - - def load_tfrecord(self, tfrecord_path: Path, sequence_id: str) -> UnifiedSequence: - """Load from Waymo TFRecord when SDK is available.""" - try: - from waymo_open_dataset import dataset_pb2 as open_dataset # type: ignore - import tensorflow as tf # type: ignore - except ImportError: - return self.load({}, sequence_id) - - frames = [] - dataset = tf.data.TFRecordDataset(str(tfrecord_path), compression_type="") - for i, raw_record in enumerate(dataset.take(10)): - frame = open_dataset.Frame() - frame.ParseFromString(bytes(raw_record.numpy())) - labels = [] - for obj in frame.laser_labels: - labels.append({ - "instance_id": str(obj.id), - "type": obj.type, - "center_x": obj.box.center_x, - "center_y": obj.box.center_y, - "center_z": obj.box.center_z, - "length": obj.box.length, - "width": obj.box.width, - "height": obj.box.height, - "heading": obj.box.heading, - }) - frames.append({ - "frame_id": f"waymo_{i:04d}", - "timestamp_us": int(frame.timestamp_micros), - "lidar_points": len(frame.lasers[0].ri_return1.range_values) if frame.lasers else 2048, - "ego_speed_kmh": 40.0, - "labels": labels, - }) - return self.load({"frames": frames}, sequence_id) diff --git a/hf-space/sensorflow/bevfusion/__init__.py b/hf-space/sensorflow/bevfusion/__init__.py deleted file mode 100644 index 51ff4f3c374bb438ba5f8af7d31d2e8d670cda26..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/bevfusion/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -"""BEV-Fusion perception engine (inspired by BEVFusion) with masklet-style -temporal propagation (inspired by SAM 3). - -What this package does ----------------------- -Fuses camera and LiDAR evidence in a unified bird's-eye-view (BEV) plane to -produce better auto-labels than a single-sensor (camera-primary) labeler, and -demonstrates the improvement with the platform's own evaluation machinery on -synthetic scenes with known ground truth. - -Concept mapping (paper -> this implementation) ----------------------------------------------- -BEVFusion: - - "Lift camera features to BEV via calibrated geometry" - -> :mod:`geometry`: pinhole projection with a 6-DoF camera-to-ego - extrinsic; monocular detections are back-projected along the viewing - ray into the BEV plane with an anisotropic (along-ray-elongated) - position covariance. - - "Rasterize LiDAR features into a BEV grid" - -> :mod:`fusion`: LiDAR detections are splatted into a metric BEV - occupancy/feature grid with tight isotropic covariance. - - "Fuse the modality feature maps in the shared BEV space" - -> :mod:`fusion.fuse_maps`: per-cell fusion of existence evidence - (noisy-OR), class evidence (reliability-weighted histograms; camera - weighted higher), and geometry evidence (inverse-variance weighted - accumulators; LiDAR dominates position where present). - - "Decode fused BEV features into 3D boxes" - -> :mod:`fusion.decode_bev`: peak extraction + ray-aware NMS + - continuous weighted decoding of center/dims/yaw/class. - -SAM 3 masklets: - - "Propagate each object's identity through time, surviving brief - disappearances without re-identification" - -> :mod:`masklet.BEVMaskletTracker`: association in the stable BEV - plane (Hungarian on BEV distance + class + size, extending - :mod:`sensorflow.temporal_tracker`), with Kalman-predicted BEV - position carrying identity across short dropouts and emitting - propagated ("masklet") boxes during occlusion gaps. - -What is simulated vs. real --------------------------- -There are NO learned features and NO neural networks here (the platform is -numpy-only by design). Camera/LiDAR "detections" are sampled from ground truth -with realistic, modality-specific failure modes (camera: occlusion/night -misses and depth ambiguity but good class discrimination; LiDAR: accurate -geometry, degraded at long range and in rain, weak class discrimination). -The fusion itself is honest geometric + probabilistic math, and the measured -improvement arises from real sensor complementarity in that math: LiDAR -existence evidence recovers camera misses, camera class evidence flows along -the viewing ray to LiDAR-anchored cells, and inverse-variance weighting gives -fused boxes near-LiDAR position accuracy. Nothing in the comparison report is -hardcoded; every number is computed at runtime from the fused outputs. -""" diff --git a/hf-space/sensorflow/bevfusion/api.py b/hf-space/sensorflow/bevfusion/api.py deleted file mode 100644 index e6b1c23aa7aa79860aae8f7f8a8836173f8a759a..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/bevfusion/api.py +++ /dev/null @@ -1,69 +0,0 @@ -"""FastAPI router for the BEV-Fusion perception engine (included by app_backend). - -Endpoints: - POST /api/bevfusion/run generate scenes, run both engines, evaluate, persist - GET /api/bevfusion/report latest comparison report (per-cohort deltas + explanations) - GET /api/bevfusion/status run inventory + engine identifiers -""" - -from __future__ import annotations - -import json -from typing import Optional - -from fastapi import APIRouter, HTTPException -from pydantic import BaseModel, Field - -from sensorflow.bevfusion import evaluate as evaluate_mod -from sensorflow.bevfusion.engines import BASELINE_ENGINE, FUSED_ENGINE - -router = APIRouter(prefix="/api/bevfusion") - - -class RunRequest(BaseModel): - n_sequences: int = Field(6, ge=1, le=24) - frames_per_sequence: int = Field(24, ge=8, le=60) - seed: int = 7 - - -@router.post("/run") -def run(req: RunRequest): - report = evaluate_mod.run_comparison( - n_sequences=req.n_sequences, - frames_per_sequence=req.frames_per_sequence, - seed=req.seed, - ) - return report - - -@router.get("/report") -def report(): - rep = evaluate_mod.latest_report() - if rep is None: - raise HTTPException(404, "No BEV-fusion comparison has been run yet; " - "POST /api/bevfusion/run first") - return rep - - -@router.get("/status") -def status(): - runs_dir = evaluate_mod.RUNS_DIR - runs = [] - if runs_dir.exists(): - for p in sorted(runs_dir.glob("bevrun-*.json")): - try: - with open(p) as f: - r = json.load(f) - runs.append({"run_id": r["run_id"], "created_at": r["created_at"], - "params": r["params"], - "recommendation": r.get("recommendation")}) - except Exception: - continue - latest: Optional[dict] = runs[-1] if runs else None - return { - "engines": {"baseline": BASELINE_ENGINE, "candidate": FUSED_ENGINE}, - "n_runs": len(runs), - "runs": runs, - "latest": latest, - "ready": latest is not None, - } diff --git a/hf-space/sensorflow/bevfusion/engines.py b/hf-space/sensorflow/bevfusion/engines.py deleted file mode 100644 index 918ed7cfbfc052e734116db0bb6b0cccf6bcf373..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/bevfusion/engines.py +++ /dev/null @@ -1,120 +0,0 @@ -"""The two auto-label engines under comparison. - -perception-v1-camera (baseline) - Camera-primary label generation: monocular detections are lifted to 3D - directly (noisy range along the ray, ground-plane z) and associated - frame-to-frame with a greedy nearest-neighbour matcher — no motion model, - no gap tolerance. Expected weaknesses (all measured, none hardcoded): - misses under occlusion and at night, position error growing with range, - ID switches/fragmentation whenever detection drops for even one frame or - the depth estimate jumps. - -perception-v3-bevfusion (candidate) - Camera + LiDAR fused in the shared BEV grid (fusion.py), decoded to boxes, - then tracked with BEV Hungarian association + masklet propagation - (masklet.py). The improvement comes from the fusion math: LiDAR existence - recovers camera misses, camera semantics label LiDAR geometry, and - inverse-variance weighting gives near-LiDAR position accuracy. -""" - -from __future__ import annotations - -import math -from typing import Dict, List - -from sensorflow.bevfusion.geometry import BEVGrid -from sensorflow.bevfusion.fusion import build_modality_map, decode_bev, fuse_maps -from sensorflow.bevfusion.masklet import BEVMaskletTracker -from sensorflow.bevfusion.scenes import SceneSequence -from sensorflow.bevfusion.sensors import ( - Detection, camera_rng, lidar_rng, simulate_camera, simulate_lidar, -) -from sensorflow.schemas.unified_frame import Object3D - -BASELINE_ENGINE = "perception-v1-camera" -FUSED_ENGINE = "perception-v3-bevfusion" - - -class FrameToFrameTracker: - """Deliberately naive baseline association: greedy NN against the previous - frame only. Any dropout or large position jitter spawns a new identity.""" - - def __init__(self, gate: float = 3.0): - self.gate = gate - self.prev: List[Dict] = [] # [{track_id, x, y}] - self.next_id = 1 - - def update(self, boxes: List[Dict]) -> List[Dict]: - used = set() - for box in sorted(boxes, key=lambda b: -b.get("confidence", 0.0)): - best, best_d = None, self.gate - for i, p in enumerate(self.prev): - if i in used: - continue - d = math.hypot(box["bbox_3d"][0] - p["x"], box["bbox_3d"][1] - p["y"]) - if d < best_d: - best, best_d = i, d - if best is not None: - used.add(best) - box["track_id"] = self.prev[best]["track_id"] - else: - box["track_id"] = self.next_id - self.next_id += 1 - self.prev = [{"track_id": b["track_id"], "x": b["bbox_3d"][0], "y": b["bbox_3d"][1]} - for b in boxes] - return boxes - - -def _det_to_box(det: Detection) -> Dict: - return { - "bbox_3d": [det.x, det.y, det.z, det.dims[0], det.dims[1], det.dims[2], det.yaw], - "class_name": det.class_name, - "class_confidence": det.class_confidence, - "confidence": det.confidence, - "propagated": False, - } - - -def run_baseline(seq: SceneSequence, seed: int, seq_index: int) -> Dict[str, List[Dict]]: - """Camera-only labels per frame. Uses the same camera detections (same rng - stream) as the fused engine, so the comparison isolates the fusion.""" - rng = camera_rng(seed, seq_index) - tracker = FrameToFrameTracker(gate=3.0) - out: Dict[str, List[Dict]] = {} - for frame in seq.frames: - boxes = [_det_to_box(d) for d in simulate_camera(frame, seq, rng)] - out[frame.frame_id] = tracker.update(boxes) - return out - - -def run_fused(seq: SceneSequence, seed: int, seq_index: int, - grid: BEVGrid = None): - """BEV-fused labels per frame, tracked with masklet propagation.""" - grid = grid or BEVGrid() - cam_gen = camera_rng(seed, seq_index) - lid_gen = lidar_rng(seed, seq_index) - tracker = BEVMaskletTracker(bounds=(grid.x_min, grid.x_max, grid.y_min, grid.y_max)) - out: Dict[str, List[Dict]] = {} - stats = {"camera_detections": 0, "lidar_detections": 0, "fused_boxes": 0} - - for frame in seq.frames: - cam_dets = simulate_camera(frame, seq, cam_gen) - lid_dets = simulate_lidar(frame, seq, lid_gen) - stats["camera_detections"] += len(cam_dets) - stats["lidar_detections"] += len(lid_dets) - - cam_map = build_modality_map(cam_dets, grid, "camera") - lid_map = build_modality_map(lid_dets, grid, "lidar") - decoded = decode_bev(fuse_maps(cam_map, lid_map)) - stats["fused_boxes"] += len(decoded) - - proposals = [Object3D(bbox_3d=b["bbox_3d"], class_name=b["class_name"], - confidence=b["confidence"]) for b in decoded] - tracked = tracker.update(frame.frame_id, proposals) - propagated_ids = getattr(tracker, "last_propagated", set()) - out[frame.frame_id] = [{ - "bbox_3d": t.bbox_3d, "class_name": t.class_name, - "confidence": t.confidence, "track_id": t.track_id, - "propagated": t.track_id in propagated_ids, - } for t in tracked] - return out, stats diff --git a/hf-space/sensorflow/bevfusion/evaluate.py b/hf-space/sensorflow/bevfusion/evaluate.py deleted file mode 100644 index 351cd007d73e299f0723924b3267abeb7375d0d5..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/bevfusion/evaluate.py +++ /dev/null @@ -1,422 +0,0 @@ -"""Self-evaluation: baseline vs BEV-fused labeler on identical scenes. - -Generates N deterministic sequences with known ground truth, runs both -engines on the *same* sensor detection streams, evaluates both against ground -truth with the platform's existing metrics (BEV IoU from -sensorflow.metrics.perception_3d; ID-switch and fragmentation rates from -sensorflow.metrics.temporal_mot; plus nuScenes-style center-distance matching, -per-cohort recall, position error and IDF1 computed here), and produces a -comparison report mirroring the shape of the megaeval compare machinery -(headline_deltas / per-cohort deltas / policy / blockers / PROMOTE -recommendation). Every number is computed at runtime from the engine outputs. - -Matching is center-distance based (2.0 m, nuScenes convention) rather than -IoU-thresholded, so the baseline's monocular depth error degrades recall and -position error smoothly instead of zeroing everything out; BEV IoU is still -reported for matched pairs. -""" - -from __future__ import annotations - -import json -import math -from datetime import datetime, timezone -from pathlib import Path -from typing import Dict, List, Optional -from uuid import uuid4 - -import numpy as np -from scipy.optimize import linear_sum_assignment - -from sensorflow.bevfusion.engines import ( - BASELINE_ENGINE, FUSED_ENGINE, run_baseline, run_fused, -) -from sensorflow.bevfusion.masklet import tracks_to_dicts -from sensorflow.bevfusion.scenes import ( - SAFETY_CLASSES, SceneSequence, generate_sequences, gt_tracks, -) -from sensorflow.metrics.perception_3d import bev_iou -from sensorflow.metrics.temporal_mot import ( - compute_id_swap_rate, compute_track_fragmentation_rate, -) - -RUNS_DIR = Path("runs/bevfusion") -MATCH_DIST_M = 2.0 - -DEFAULT_POLICY = { - "max_recall_drop": 0.010, - "max_precision_drop": 0.015, - "max_safety_recall_drop": 0.005, - "max_cohort_recall_drop": 0.050, - "min_cohort_support": 20, -} - -COHORT_EXPLANATIONS = { - "overall": "Fused existence evidence (noisy-OR of camera and LiDAR) plus " - "inverse-variance geometry gives higher recall and near-LiDAR " - "position accuracy wherever either sensor fires.", - "day": "Both sensors healthy: fusion mainly sharpens positions " - "(LiDAR-dominated inverse-variance weighting) and suppresses " - "single-modality ghosts at the decode threshold.", - "night": "The camera misses heavily in low light; LiDAR is unaffected by " - "darkness, so LiDAR existence evidence in the BEV grid recovers " - "the missed objects and the class comes from LiDAR geometry " - "templates or temporal continuity.", - "clear": "Nominal conditions; improvement is dominated by monocular depth " - "error being replaced by LiDAR geometry.", - "rain": "Rain degrades LiDAR at range (spray attenuation) and the camera " - "mildly; each modality covers part of the other's losses, so the " - "fused map keeps existence evidence where either survives.", - "occluded": "Camera line of sight is blocked; LiDAR (mounted higher, " - "sensing 3D structure) keeps partial returns, and masklet " - "propagation carries identity and a predicted box through the " - "remaining gap frames.", - "long_range": "Beyond ~45 m LiDAR thins out and monocular depth error " - "exceeds the match gate; fusion helps where LiDAR still " - "fires, and camera bearing keeps existence alive elsewhere.", -} - - -# ------------------------------------------------------------------ matching - - -def _match_frame(pred_boxes: List[Dict], gt_boxes: List, dist_m: float = MATCH_DIST_M): - """Greedy center-distance matching. Returns (pairs, unmatched_pred_idx).""" - cands = [] - for pi, p in enumerate(pred_boxes): - for gi, g in enumerate(gt_boxes): - d = math.hypot(p["bbox_3d"][0] - g.bbox_3d[0], p["bbox_3d"][1] - g.bbox_3d[1]) - if d <= dist_m: - cands.append((d, pi, gi)) - cands.sort() - used_p, used_g, pairs = set(), set(), [] - for d, pi, gi in cands: - if pi in used_p or gi in used_g: - continue - pairs.append((pi, gi, d)) - used_p.add(pi) - used_g.add(gi) - unmatched_pred = [i for i in range(len(pred_boxes)) if i not in used_p] - return pairs, unmatched_pred - - -def _gt_cohorts(gt, seq: SceneSequence) -> List[str]: - cohorts = ["overall", seq.time_of_day, seq.weather] - if gt.occluded: - cohorts.append("occluded") - if gt.distance > 40.0: - cohorts.append("long_range") - return cohorts - - -# ------------------------------------------------------------------ IDF1 - - -def compute_idf1(pred_tracks: List[Dict], gt_tracks_list: List[Dict], - dist_m: float = MATCH_DIST_M) -> float: - """IDF1 via optimal 1:1 track assignment (Hungarian on frame overlaps).""" - if not pred_tracks or not gt_tracks_list: - return 0.0 - n_pred = sum(len(t["frames"]) for t in pred_tracks) - n_gt = sum(len(t["frames"]) for t in gt_tracks_list) - overlap = np.zeros((len(pred_tracks), len(gt_tracks_list))) - for i, pt in enumerate(pred_tracks): - pframes = {f["frame_id"]: f["bbox_3d"] for f in pt["frames"]} - for j, gt in enumerate(gt_tracks_list): - n = 0 - for f in gt["frames"]: - pb = pframes.get(f["frame_id"]) - if pb is not None and math.hypot(pb[0] - f["bbox_3d"][0], - pb[1] - f["bbox_3d"][1]) <= dist_m: - n += 1 - overlap[i, j] = n - rows, cols = linear_sum_assignment(-overlap) - idtp = float(overlap[rows, cols].sum()) - return round(2.0 * idtp / max(n_pred + n_gt, 1), 4) - - -# ------------------------------------------------------------------ engine evaluation - - -def evaluate_engine(sequences: List[SceneSequence], - preds_by_seq: Dict[str, Dict[str, List[Dict]]]) -> Dict: - """Evaluate one engine's per-frame labels against ground truth.""" - cohort_stats: Dict[str, Dict] = {} - class_stats: Dict[str, Dict] = {} - tp = fp = fn = 0 - ious: List[float] = [] - pos_errors: List[float] = [] - class_correct = class_total = 0 - safety_tp = safety_fn = 0 - - def _coh(name): - return cohort_stats.setdefault(name, {"n_gt": 0, "matched": 0, - "pos_errors": [], "ious": [], "fp": 0}) - - for seq in sequences: - preds = preds_by_seq[seq.sequence_id] - for frame in seq.frames: - pred_boxes = preds.get(frame.frame_id, []) - pairs, unmatched_pred = _match_frame(pred_boxes, frame.gt) - matched_gt = {gi for _, gi, _ in pairs} - - for pi, gi, d in pairs: - p, g = pred_boxes[pi], frame.gt[gi] - tp += 1 - iou = bev_iou(p["bbox_3d"], g.bbox_3d) - ious.append(iou) - pos_errors.append(d) - class_total += 1 - if p["class_name"] == g.class_name: - class_correct += 1 - if g.class_name in SAFETY_CLASSES: - safety_tp += 1 - cs = class_stats.setdefault(g.class_name, {"n_gt": 0, "matched": 0}) - cs["n_gt"] += 1 - cs["matched"] += 1 - for name in _gt_cohorts(g, seq): - c = _coh(name) - c["n_gt"] += 1 - c["matched"] += 1 - c["pos_errors"].append(d) - c["ious"].append(iou) - - for gi, g in enumerate(frame.gt): - if gi in matched_gt: - continue - fn += 1 - if g.class_name in SAFETY_CLASSES: - safety_fn += 1 - class_stats.setdefault(g.class_name, {"n_gt": 0, "matched": 0})["n_gt"] += 1 - for name in _gt_cohorts(g, seq): - _coh(name)["n_gt"] += 1 - - for pi in unmatched_pred: - fp += 1 - _coh("overall")["fp"] += 1 - _coh(seq.time_of_day)["fp"] += 1 - _coh(seq.weather)["fp"] += 1 - - precision = tp / (tp + fp) if tp + fp else 0.0 - recall = tp / (tp + fn) if tp + fn else 0.0 - f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 - - # Tracking metrics via the platform's temporal MOT module + IDF1. - # Track ids restart per sequence (one tracker per sequence), so namespace - # them with the sequence id before pooling. - pred_tracks: List[Dict] = [] - for seq in sequences: - for t in tracks_to_dicts(preds_by_seq[seq.sequence_id]): - t["track_id"] = f"{seq.sequence_id}:{t['track_id']}" - pred_tracks.append(t) - gts = gt_tracks(sequences) - id_switch_rate = compute_id_swap_rate(pred_tracks, gts) - frag_rate = compute_track_fragmentation_rate(pred_tracks, gts) - idf1 = compute_idf1(pred_tracks, gts) - - cohorts = {} - for name, c in cohort_stats.items(): - cohorts[name] = { - "n_gt": c["n_gt"], - "recall": round(c["matched"] / c["n_gt"], 4) if c["n_gt"] else None, - "position_error_m": round(float(np.mean(c["pos_errors"])), 4) if c["pos_errors"] else None, - "mean_iou": round(float(np.mean(c["ious"])), 4) if c["ious"] else None, - "false_positives": c["fp"], - } - per_class = { - cls: {"n_gt": s["n_gt"], - "recall": round(s["matched"] / s["n_gt"], 4) if s["n_gt"] else None} - for cls, s in sorted(class_stats.items()) - } - - return { - "headline": { - "precision": round(precision, 4), - "recall": round(recall, 4), - "f1": round(f1, 4), - "mean_iou": round(float(np.mean(ious)), 4) if ious else 0.0, - "position_error_m": round(float(np.mean(pos_errors)), 4) if pos_errors else None, - "class_accuracy": round(class_correct / class_total, 4) if class_total else None, - "safety_recall": round(safety_tp / (safety_tp + safety_fn), 4) - if safety_tp + safety_fn else None, - "id_switch_rate": round(id_switch_rate, 4), - "fragmentation_rate": round(frag_rate, 4), - "idf1": idf1, - }, - "counts": {"tp": tp, "fp": fp, "fn": fn, - "pred_tracks": len(pred_tracks), "gt_tracks": len(gts)}, - "cohorts": cohorts, - "per_class": per_class, - } - - -# ------------------------------------------------------------------ comparison report - -# Metrics where lower is better. -_LOWER_IS_BETTER = {"position_error_m", "id_switch_rate", "fragmentation_rate"} -_HEADLINE_ORDER = ["precision", "recall", "f1", "mean_iou", "position_error_m", - "class_accuracy", "safety_recall", "id_switch_rate", - "fragmentation_rate", "idf1"] - - -def build_comparison(base_eval: Dict, cand_eval: Dict, - policy: Optional[Dict] = None) -> Dict: - """Comparison report mirroring the megaeval compare shape - (headline_deltas / per-cohort / policy / blockers / recommendation).""" - policy = {**DEFAULT_POLICY, **(policy or {})} - bh, ch = base_eval["headline"], cand_eval["headline"] - - headline_deltas = [] - for m in _HEADLINE_ORDER: - b, c = bh.get(m), ch.get(m) - if b is None or c is None: - continue - delta = round(c - b, 4) - headline_deltas.append({ - "metric": m, "baseline": b, "candidate": c, "delta": delta, - "improved": (delta < 0) if m in _LOWER_IS_BETTER else (delta > 0), - }) - - per_cohort = [] - for name, b in base_eval["cohorts"].items(): - c = cand_eval["cohorts"].get(name) - if not c or b["recall"] is None or c["recall"] is None: - continue - per_cohort.append({ - "cohort": name, - "n_gt": c["n_gt"], - "recall_baseline": b["recall"], "recall_candidate": c["recall"], - "recall_delta": round(c["recall"] - b["recall"], 4), - "position_error_baseline_m": b["position_error_m"], - "position_error_candidate_m": c["position_error_m"], - "mean_iou_baseline": b["mean_iou"], "mean_iou_candidate": c["mean_iou"], - "explanation": COHORT_EXPLANATIONS.get(name, ""), - }) - per_cohort.sort(key=lambda x: x["recall_delta"], reverse=True) - - per_class = [] - for cls, b in base_eval["per_class"].items(): - c = cand_eval["per_class"].get(cls) - if not c or b["recall"] is None or c["recall"] is None: - continue - per_class.append({"class": cls, "n_gt": c["n_gt"], - "recall_baseline": b["recall"], - "recall_candidate": c["recall"], - "recall_delta": round(c["recall"] - b["recall"], 4)}) - - hd = {h["metric"]: h["delta"] for h in headline_deltas} - blockers: List[str] = [] - if hd.get("recall", 0) < -policy["max_recall_drop"]: - blockers.append(f"headline recall dropped {hd['recall']:+.4f}") - if hd.get("precision", 0) < -policy["max_precision_drop"]: - blockers.append(f"headline precision dropped {hd['precision']:+.4f}") - if hd.get("safety_recall", 0) < -policy["max_safety_recall_drop"]: - blockers.append(f"safety recall dropped {hd['safety_recall']:+.4f}") - for c in per_cohort: - if c["n_gt"] >= policy["min_cohort_support"] and \ - c["recall_delta"] < -policy["max_cohort_recall_drop"]: - blockers.append( - f"REGRESSION: cohort {c['cohort']} recall " - f"{c['recall_baseline']*100:.1f}% -> {c['recall_candidate']*100:.1f}%") - - improvements = [f"{h['metric']}: {h['baseline']} -> {h['candidate']} " - f"({h['delta']:+.4f})" for h in headline_deltas if h["improved"]] - - return { - "headline_deltas": headline_deltas, - "per_cohort": per_cohort, - "per_class": per_class, - "policy": policy, - "blockers": blockers, - "improvements": improvements, - "recommendation": "DO_NOT_PROMOTE" if blockers else "PROMOTE", - } - - -# ------------------------------------------------------------------ orchestration - - -def run_comparison(n_sequences: int = 6, frames_per_sequence: int = 24, - seed: int = 7, out_dir: Optional[Path] = None, - persist: bool = True) -> Dict: - """Generate scenes, run both engines, evaluate both, build + persist report.""" - sequences = generate_sequences(n_sequences=n_sequences, - frames_per_sequence=frames_per_sequence, seed=seed) - - baseline_preds: Dict[str, Dict[str, List[Dict]]] = {} - fused_preds: Dict[str, Dict[str, List[Dict]]] = {} - fusion_stats = {"camera_detections": 0, "lidar_detections": 0, "fused_boxes": 0} - for qi, seq in enumerate(sequences): - baseline_preds[seq.sequence_id] = run_baseline(seq, seed, qi) - fused, stats = run_fused(seq, seed, qi) - fused_preds[seq.sequence_id] = fused - for k in fusion_stats: - fusion_stats[k] += stats[k] - - base_eval = evaluate_engine(sequences, baseline_preds) - cand_eval = evaluate_engine(sequences, fused_preds) - comparison = build_comparison(base_eval, cand_eval) - - n_frames = sum(len(s.frames) for s in sequences) - n_gt = sum(len(f.gt) for s in sequences for f in s.frames) - process_units = _record_process_units(fusion_stats, n_frames, n_gt) - - report = { - "run_id": f"bevrun-{uuid4().hex[:10]}", - "created_at": datetime.now(timezone.utc).isoformat(), - "params": {"n_sequences": n_sequences, - "frames_per_sequence": frames_per_sequence, "seed": seed}, - "engines": {"baseline": BASELINE_ENGINE, "candidate": FUSED_ENGINE}, - "scale": {"frames": n_frames, "gt_boxes": n_gt, **fusion_stats}, - "baseline": base_eval, - "candidate": cand_eval, - **comparison, - "process_units": process_units, - "notes": ( - "Honest simulation: no learned features; detections are sampled from " - "ground truth with modality-specific failure models (camera: occlusion/" - "night misses + monocular depth ambiguity, good semantics; LiDAR: " - "accurate geometry, range/rain degradation, weak semantics). The " - "improvement arises at runtime from geometric+probabilistic fusion in " - "the BEV grid and masklet propagation — sensor complementarity is real." - ), - } - - if persist: - out = Path(out_dir) if out_dir else RUNS_DIR - out.mkdir(parents=True, exist_ok=True) - with open(out / f"{report['run_id']}.json", "w") as f: - json.dump(report, f, indent=2) - with open(out / "latest.json", "w") as f: - json.dump(report, f, indent=2) - return report - - -def _record_process_units(fusion_stats: Dict, n_frames: int, n_gt: int) -> Dict: - """Record fusion-stage process units via the platform's accounting - (best effort: skipped silently if the evaluation store is unavailable).""" - units = {} - try: - from sensorflow.evaluation.process_units import ProcessMeter - from sensorflow.evaluation.records import get_store - meter = ProcessMeter(get_store(), run_id="bevfusion") - units["bev_camera_projection"] = meter.record( - "bev_camera_projection", fusion_stats["camera_detections"], factor=0.2) - units["bev_lidar_rasterization"] = meter.record( - "bev_lidar_rasterization", fusion_stats["lidar_detections"], factor=0.2) - units["bev_fusion"] = meter.record("bev_fusion", n_frames, factor=0.8) - units["masklet_tracking"] = meter.record( - "masklet_tracking", fusion_stats["fused_boxes"], factor=0.3) - units["label_evaluation"] = meter.record("label_evaluation", n_gt, factor=0.5) - units["total"] = sum(units.values()) - except Exception: - units = {"total": 0, "note": "process accounting unavailable"} - return units - - -def latest_report(out_dir: Optional[Path] = None) -> Optional[Dict]: - path = (Path(out_dir) if out_dir else RUNS_DIR) / "latest.json" - if not path.exists(): - return None - with open(path) as f: - return json.load(f) diff --git a/hf-space/sensorflow/bevfusion/fusion.py b/hf-space/sensorflow/bevfusion/fusion.py deleted file mode 100644 index fe49cb20a61ce2724af5c7fee5485253a4a52d3a..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/bevfusion/fusion.py +++ /dev/null @@ -1,275 +0,0 @@ -"""BEV rasterization, per-cell modality fusion, and box decoding. - -This is the numpy analogue of BEVFusion's shared-BEV-space fusion: - -1. Each modality's detections are splatted into a metric BEV feature map - (:class:`ModalityBEV`) as Gaussian evidence. The splat covariance is the - detection's position covariance (floored at the cell size for existence so - tight LiDAR kernels are not lost between cell centers). Three kinds of - evidence are accumulated per cell: - - - existence: noisy-OR of (confidence x kernel) — "is something here?"; - - geometry: inverse-variance weighted votes for the detection's continuous - mean/dims/yaw (weight = confidence x kernel / worst-axis variance), so - LiDAR dominates position wherever it has evidence and there is no grid - quantization in the decoded centers; - - class: reliability-weighted class histograms. Camera class evidence is - splatted with an inflated along-ray kernel: monocular semantics is - positionally vague along the ray, which is precisely what lets a camera - label flow to the LiDAR-anchored cell of the same object even when the - camera's depth estimate is metres off. - -2. :func:`fuse_maps` fuses per cell: noisy-OR for existence, summed - inverse-variance geometry accumulators, summed weighted class histograms - (camera weight 1.0 vs LiDAR 0.35 — camera is the semantics expert). - -3. :func:`decode_bev` extracts local existence maxima above a threshold, - applies euclidean NMS plus ray-aware NMS (two peaks on the same bearing - within the monocular depth-ambiguity band are one object — the fused, - LiDAR-anchored peak wins over the camera's displaced ghost), then decodes - each surviving peak's neighbourhood into a 3D box with class + confidence. -""" - -from __future__ import annotations - -import math -from dataclasses import dataclass, field -from typing import Dict, List, Optional - -import numpy as np -from scipy.ndimage import maximum_filter - -from sensorflow.bevfusion.geometry import BEVGrid -from sensorflow.bevfusion.scenes import CLASSES -from sensorflow.bevfusion.sensors import Detection - -CLASS_INDEX = {c: i for i, c in enumerate(CLASSES)} - -# Modality reliability weights for class evidence (camera >> lidar). -CLASS_WEIGHT = {"camera": 1.0, "lidar": 0.35} -# Camera class evidence is smeared further along the ray (see module docstring). -CAMERA_CLASS_SIGMA_SCALE = 2.0 - - -@dataclass -class ModalityBEV: - """Per-modality BEV feature map built by splatting detections.""" - - grid: BEVGrid - modality: str = "camera" - miss: np.ndarray = field(default=None) # prod(1 - c*g): existence complement - sum_w: np.ndarray = field(default=None) - sum_wx: np.ndarray = field(default=None) - sum_wy: np.ndarray = field(default=None) - sum_wz: np.ndarray = field(default=None) - sum_wl: np.ndarray = field(default=None) - sum_ww: np.ndarray = field(default=None) - sum_wh: np.ndarray = field(default=None) - sum_wsin: np.ndarray = field(default=None) - sum_wcos: np.ndarray = field(default=None) - class_hist: np.ndarray = field(default=None) # (nx, ny, n_classes) - - def __post_init__(self): - nx, ny = self.grid.nx, self.grid.ny - self.miss = np.ones((nx, ny)) - for name in ("sum_w", "sum_wx", "sum_wy", "sum_wz", "sum_wl", "sum_ww", - "sum_wh", "sum_wsin", "sum_wcos"): - setattr(self, name, np.zeros((nx, ny))) - self.class_hist = np.zeros((nx, ny, len(CLASSES))) - - @property - def existence(self) -> np.ndarray: - return 1.0 - self.miss - - # -- splatting - - def _window(self, mx: float, my: float, ex: float, ey: float): - g = self.grid - ix0 = max(0, int((mx - ex - g.x_min) / g.cell)) - ix1 = min(g.nx, int((mx + ex - g.x_min) / g.cell) + 1) - iy0 = max(0, int((my - ey - g.y_min) / g.cell)) - iy1 = min(g.ny, int((my + ey - g.y_min) / g.cell) + 1) - if ix0 >= ix1 or iy0 >= iy1: - return None - xs = g.x_min + (np.arange(ix0, ix1) + 0.5) * g.cell - ys = g.y_min + (np.arange(iy0, iy1) + 0.5) * g.cell - X, Y = np.meshgrid(xs, ys, indexing="ij") - return (slice(ix0, ix1), slice(iy0, iy1)), X - mx, Y - my - - @staticmethod - def _kernel(dx: np.ndarray, dy: np.ndarray, cov: np.ndarray) -> np.ndarray: - inv = np.linalg.inv(cov) - q = inv[0, 0] * dx * dx + (inv[0, 1] + inv[1, 0]) * dx * dy + inv[1, 1] * dy * dy - return np.exp(-0.5 * q) - - def splat(self, det: Detection) -> None: - cov = np.array(det.cov, dtype=float) - # Floor the existence/geometry kernel at ~the cell size so tight LiDAR - # Gaussians are not lost between cell centers (grid smoothing only: - # geometry votes still use the true covariance for their weights). - floor = (0.8 * self.grid.cell) ** 2 - cov_splat = cov + np.eye(2) * floor - ex = 3.0 * math.sqrt(cov_splat[0, 0]) - ey = 3.0 * math.sqrt(cov_splat[1, 1]) - win = self._window(det.x, det.y, ex, ey) - if win is None: - return - sl, dx, dy = win - g = self._kernel(dx, dy, cov_splat) - - # Existence: noisy-OR accumulation. - self.miss[sl] *= (1.0 - det.confidence * g) - - # Geometry: inverse-variance weighted votes for the *continuous* mean. - # The kernel is normalized to unit mass so each detection contributes - # exactly one measurement of total weight conf/sigma_eff^2 no matter - # how widely its kernel is spread — proper inverse-variance fusion. - eigvals = np.linalg.eigvalsh(cov) - sigma_eff_sq = float(max(eigvals[-1], 1e-4)) # worst-axis variance - g_mass = float(g.sum()) - if g_mass <= 1e-12: - return - w = det.confidence * (g / g_mass) / sigma_eff_sq - self.sum_w[sl] += w - self.sum_wx[sl] += w * det.x - self.sum_wy[sl] += w * det.y - self.sum_wz[sl] += w * det.z - self.sum_wl[sl] += w * det.dims[0] - self.sum_ww[sl] += w * det.dims[1] - self.sum_wh[sl] += w * det.dims[2] - self.sum_wsin[sl] += w * math.sin(det.yaw) - self.sum_wcos[sl] += w * math.cos(det.yaw) - - # Class: reliability-weighted histogram; camera semantics smeared - # further along the ray so labels reach the LiDAR-anchored cell. - cls_idx = CLASS_INDEX.get(det.class_name) - if cls_idx is None: - return - if self.modality == "camera": - cov_cls = cov * (CAMERA_CLASS_SIGMA_SCALE ** 2) + np.eye(2) * floor - else: - cov_cls = cov * 4.0 + np.eye(2) * floor - exc = 3.0 * math.sqrt(cov_cls[0, 0]) - eyc = 3.0 * math.sqrt(cov_cls[1, 1]) - winc = self._window(det.x, det.y, exc, eyc) - if winc is None: - return - slc, dxc, dyc = winc - gc = self._kernel(dxc, dyc, cov_cls) - wc = det.confidence * det.class_confidence * CLASS_WEIGHT.get(self.modality, 0.5) - self.class_hist[slc[0], slc[1], cls_idx] += wc * gc - - -def build_modality_map(dets: List[Detection], grid: BEVGrid, modality: str) -> ModalityBEV: - bev = ModalityBEV(grid=grid, modality=modality) - for det in dets: - if det.modality == modality: - bev.splat(det) - return bev - - -@dataclass -class FusedBEV: - grid: BEVGrid - existence: np.ndarray - sum_w: np.ndarray - sum_wx: np.ndarray - sum_wy: np.ndarray - sum_wz: np.ndarray - sum_wl: np.ndarray - sum_ww: np.ndarray - sum_wh: np.ndarray - sum_wsin: np.ndarray - sum_wcos: np.ndarray - class_hist: np.ndarray - - -def fuse_maps(cam: ModalityBEV, lidar: ModalityBEV) -> FusedBEV: - """Per-cell confidence-weighted fusion of the two modality feature maps.""" - return FusedBEV( - grid=cam.grid, - existence=1.0 - cam.miss * lidar.miss, # noisy-OR across modalities - sum_w=cam.sum_w + lidar.sum_w, - sum_wx=cam.sum_wx + lidar.sum_wx, - sum_wy=cam.sum_wy + lidar.sum_wy, - sum_wz=cam.sum_wz + lidar.sum_wz, - sum_wl=cam.sum_wl + lidar.sum_wl, - sum_ww=cam.sum_ww + lidar.sum_ww, - sum_wh=cam.sum_wh + lidar.sum_wh, - sum_wsin=cam.sum_wsin + lidar.sum_wsin, - sum_wcos=cam.sum_wcos + lidar.sum_wcos, - class_hist=cam.class_hist + lidar.class_hist, - ) - - -def decode_bev(fused: FusedBEV, threshold: float = 0.30, - nms_radius_m: float = 2.0, gather_radius_m: float = 2.0, - ray_nms_bearing_deg: float = 1.5, - ray_nms_range_m: float = 6.0) -> List[Dict]: - """Decode fused BEV cells into 3D boxes with class + confidence.""" - grid = fused.grid - e = fused.existence - local_max = (e >= maximum_filter(e, size=5)) & (e >= threshold) - ixs, iys = np.nonzero(local_max) - if ixs.size == 0: - return [] - order = np.argsort(-e[ixs, iys]) - peaks = [(int(ixs[i]), int(iys[i])) for i in order] - - accepted: List[Dict] = [] - gather_cells = max(1, int(round(gather_radius_m / grid.cell))) - for ix, iy in peaks: - cx, cy = grid.center(ix, iy) - suppress = False - for box in accepted: - bx, by = box["_peak_xy"] - if math.hypot(cx - bx, cy - by) < nms_radius_m: - suppress = True - break - # Ray-aware NMS: same bearing, within the monocular depth - # ambiguity band -> the weaker peak is the camera's range ghost. - db = abs(math.atan2(cy, cx) - math.atan2(by, bx)) - db = min(db, 2 * math.pi - db) - if math.degrees(db) < ray_nms_bearing_deg and \ - abs(math.hypot(cx, cy) - math.hypot(bx, by)) < ray_nms_range_m: - suppress = True - break - if suppress: - continue - - x0, x1 = max(0, ix - gather_cells), min(grid.nx, ix + gather_cells + 1) - y0, y1 = max(0, iy - gather_cells), min(grid.ny, iy + gather_cells + 1) - sl = (slice(x0, x1), slice(y0, y1)) - w = float(fused.sum_w[sl].sum()) - if w <= 1e-9: - continue - x = float(fused.sum_wx[sl].sum() / w) - y = float(fused.sum_wy[sl].sum() / w) - z = float(fused.sum_wz[sl].sum() / w) - l = float(fused.sum_wl[sl].sum() / w) - wd = float(fused.sum_ww[sl].sum() / w) - h = float(fused.sum_wh[sl].sum() / w) - yaw = float(math.atan2(fused.sum_wsin[sl].sum(), fused.sum_wcos[sl].sum())) - hist = fused.class_hist[sl].sum(axis=(0, 1)) - hist_total = float(hist.sum()) - cls_idx = int(np.argmax(hist)) if hist_total > 0 else 0 - accepted.append({ - "bbox_3d": [x, y, z, l, wd, h, yaw], - "class_name": CLASSES[cls_idx], - "class_confidence": round(float(hist[cls_idx] / hist_total), 4) if hist_total > 0 else 0.0, - "confidence": round(float(e[ix, iy]), 4), - "_peak_xy": (cx, cy), - }) - - for box in accepted: - box.pop("_peak_xy", None) - return accepted - - -def fuse_frame(camera_dets: List[Detection], lidar_dets: List[Detection], - grid: Optional[BEVGrid] = None, threshold: float = 0.30) -> List[Dict]: - """Convenience: full rasterize -> fuse -> decode for one frame.""" - grid = grid or BEVGrid() - cam_map = build_modality_map(camera_dets, grid, "camera") - lidar_map = build_modality_map(lidar_dets, grid, "lidar") - return decode_bev(fuse_maps(cam_map, lidar_map), threshold=threshold) diff --git a/hf-space/sensorflow/bevfusion/geometry.py b/hf-space/sensorflow/bevfusion/geometry.py deleted file mode 100644 index 3fc3e030910d92c75a2f7f5f14542a5c4d520072..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/bevfusion/geometry.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Calibrated camera geometry + metric BEV grid. - -Coordinate conventions (matching the rest of the platform, e.g. -sensorflow/evaluation/synthetic.py): ego frame has x forward (m), y left (m), -z up (m), ground plane at z=0. - -The camera uses the standard optical convention (z forward, x right, y down); -`make_camera_extrinsic` builds the 6-DoF camera-to-ego transform, mirroring -the Calibration.camera_to_ego 4x4 matrices in -sensorflow/schemas/unified_frame.py. -""" - -from __future__ import annotations - -import math -from dataclasses import dataclass -from typing import Optional, Tuple - -import numpy as np - -# Default intrinsics, consistent with the existing synthetic generator -# (sensorflow/evaluation/synthetic.py: CAM_W, CAM_H, CAM_F). -CAM_W, CAM_H, CAM_F = 800, 450, 500.0 - - -def make_intrinsics(fx: float = CAM_F, fy: float = CAM_F, - cx: float = CAM_W / 2, cy: float = CAM_H / 2) -> np.ndarray: - return np.array([[fx, 0.0, cx], [0.0, fy, cy], [0.0, 0.0, 1.0]]) - - -def make_camera_extrinsic(tx: float = 0.2, ty: float = 0.0, tz: float = 1.5, - yaw: float = 0.0) -> np.ndarray: - """4x4 camera-to-ego transform for a forward-looking camera. - - Camera axes expressed in the ego frame (before yaw): optical axis - z_cam -> +x_ego, x_cam (image right) -> -y_ego, y_cam (image down) -> -z_ego. - """ - base = np.array([ - [0.0, 0.0, 1.0], # ego x row: from cam z - [-1.0, 0.0, 0.0], # ego y row: from -cam x - [0.0, -1.0, 0.0], # ego z row: from -cam y - ]) - c, s = math.cos(yaw), math.sin(yaw) - rz = np.array([[c, -s, 0.0], [s, c, 0.0], [0.0, 0.0, 1.0]]) - T = np.eye(4) - T[:3, :3] = rz @ base - T[:3, 3] = [tx, ty, tz] - return T - - -def project_to_image(point_ego: np.ndarray, cam_to_ego: np.ndarray, - K: np.ndarray) -> Optional[Tuple[float, float, float]]: - """Ego-frame point -> (u, v, range). None if behind the camera.""" - R, t = cam_to_ego[:3, :3], cam_to_ego[:3, 3] - p_cam = R.T @ (np.asarray(point_ego, dtype=float) - t) - if p_cam[2] < 0.5: # behind or too close to the optical center - return None - u = K[0, 0] * p_cam[0] / p_cam[2] + K[0, 2] - v = K[1, 1] * p_cam[1] / p_cam[2] + K[1, 2] - rng = float(np.linalg.norm(p_cam)) - return float(u), float(v), rng - - -def backproject(u: float, v: float, range_est: float, cam_to_ego: np.ndarray, - K: np.ndarray) -> np.ndarray: - """Pixel + estimated range along the viewing ray -> ego-frame 3D point. - - This is the monocular "lift" step: the bearing (pixel) is precise, the - range is whatever the depth estimate says — hence the along-ray - uncertainty that the BEV fusion exploits. - """ - d_cam = np.array([(u - K[0, 2]) / K[0, 0], (v - K[1, 2]) / K[1, 1], 1.0]) - d_cam /= np.linalg.norm(d_cam) - R, t = cam_to_ego[:3, :3], cam_to_ego[:3, 3] - return R @ (d_cam * range_est) + t - - -@dataclass(frozen=True) -class BEVGrid: - """Metric BEV grid over the ego frame (x forward, y left).""" - - x_min: float = 0.0 - x_max: float = 80.0 - y_min: float = -25.0 - y_max: float = 25.0 - cell: float = 0.5 - - @property - def nx(self) -> int: - return int(round((self.x_max - self.x_min) / self.cell)) - - @property - def ny(self) -> int: - return int(round((self.y_max - self.y_min) / self.cell)) - - def index(self, x: float, y: float) -> Optional[Tuple[int, int]]: - ix = int((x - self.x_min) / self.cell) - iy = int((y - self.y_min) / self.cell) - if 0 <= ix < self.nx and 0 <= iy < self.ny: - return ix, iy - return None - - def center(self, ix: int, iy: int) -> Tuple[float, float]: - return (self.x_min + (ix + 0.5) * self.cell, - self.y_min + (iy + 0.5) * self.cell) - - def contains(self, x: float, y: float) -> bool: - return self.x_min <= x < self.x_max and self.y_min <= y < self.y_max diff --git a/hf-space/sensorflow/bevfusion/masklet.py b/hf-space/sensorflow/bevfusion/masklet.py deleted file mode 100644 index 6c09d3967dfc91a1a5d5219e9271404a13880b2a..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/bevfusion/masklet.py +++ /dev/null @@ -1,150 +0,0 @@ -"""Masklet-style temporal propagation with BEV association. - -Extends the platform's :class:`sensorflow.temporal_tracker.TemporalTracker` -(Kalman + Hungarian) with the two ideas this feature needs, mapping to SAM 3's -masklet propagation: - -- association happens in the stable metric BEV plane with a richer cost: - BEV center distance + class-mismatch penalty + size-difference penalty - (fused boxes have reliable LiDAR geometry, so size/class gating is safe); -- masklet propagation: a track whose object briefly disappears (e.g. a 2-3 - frame camera occlusion that LiDAR also partially loses) keeps its identity - alive via the Kalman-predicted BEV position and *emits propagated boxes* - for the gap frames (confidence-decayed, flagged ``propagated``), so brief - dropouts are bridged without spawning new IDs and without fragmenting the - track — the direct analogue of a masklet being carried through frames where - the object is not re-detected. - -The baseline camera labeler deliberately uses frame-to-frame greedy -association with no motion model and no gap tolerance (see engines.py), which -is what makes ID switches and fragmentation measurable. -""" - -from __future__ import annotations - -import math -from typing import Dict, List, Tuple - -import numpy as np - -from sensorflow.schemas.unified_frame import Object3D -from sensorflow.temporal_tracker import TemporalTracker - - -class BEVMaskletTracker(TemporalTracker): - """BEV Hungarian association + masklet propagation across dropouts.""" - - def __init__(self, max_age: int = 6, distance_gate: float = 4.0, - propagate_max: int = 3, min_hits_to_propagate: int = 2, - class_penalty: float = 1.5, size_penalty: float = 0.3, - dt: float = 0.1, - bounds: Tuple[float, float, float, float] = (0.0, 80.0, -25.0, 25.0)): - super().__init__(max_age=max_age, distance_gate=distance_gate) - self.dt = dt - self.propagate_max = propagate_max - self.min_hits_to_propagate = min_hits_to_propagate - self.class_penalty = class_penalty - self.size_penalty = size_penalty - self.bounds = bounds - - # -- association: BEV distance + class + size (replaces the parent cost) - - def _associate(self, proposals: List[Object3D]) -> List[Tuple[int, int]]: - from scipy.optimize import linear_sum_assignment - - track_ids = list(self.tracks.keys()) - cost = np.full((len(track_ids), len(proposals)), 1e6) - for ti, tid in enumerate(track_ids): - track = self.tracks[tid] - last_dims = track.history[-1]["bbox_3d"][3:6] if track.history else [4.5, 1.9, 1.6] - for pi, prop in enumerate(proposals): - dist = math.hypot(track.state[0] - prop.bbox_3d[0], - track.state[1] - prop.bbox_3d[1]) - if dist >= self.distance_gate: - continue - c = dist - if track.class_name != prop.class_name: - c += self.class_penalty - c += self.size_penalty * (abs(last_dims[0] - prop.bbox_3d[3]) - + abs(last_dims[1] - prop.bbox_3d[4])) - cost[ti, pi] = c - - row_ind, col_ind = linear_sum_assignment(cost) - return [(track_ids[r], c) for r, c in zip(row_ind, col_ind) - if cost[r, c] < 1e6] - - # -- update with masklet propagation - - def update(self, frame_id: str, proposals: List[Object3D]) -> List[Object3D]: - out = super().update(frame_id, proposals) - self.last_propagated: set = set() - - # The parent Kalman observes position only, so its velocity estimate - # converges too slowly to bridge gaps for fast objects. Blend in a - # finite-difference velocity from the last two *real* measurements - # (SORT-style), which the constant-velocity prediction then carries - # through occlusion gaps. - for track in self.tracks.values(): - if track.missed_frames != 0: - continue # not matched this frame - real_idx = [i for i, h in enumerate(track.history) if not h.get("propagated")] - if len(real_idx) >= 2: - i1, i0 = real_idx[-1], real_idx[-2] - span = max(i1 - i0, 1) # frames elapsed (propagated entries fill gaps) - dx = track.history[i1]["bbox_3d"][0] - track.history[i0]["bbox_3d"][0] - dy = track.history[i1]["bbox_3d"][1] - track.history[i0]["bbox_3d"][1] - v_meas = np.array([dx, dy]) / (span * self.dt) - track.state[2:4] = 0.5 * track.state[2:4] + 0.5 * v_meas - - # Track-level class voting: identity persistence lets high-confidence - # camera semantics (or the modal LiDAR template guess) stabilize the - # per-frame class of every box the track emits. - if not hasattr(self, "_class_votes"): - self._class_votes: Dict[int, Dict[str, float]] = {} - for prop in out: - votes = self._class_votes.setdefault(prop.track_id, {}) - votes[prop.class_name] = votes.get(prop.class_name, 0.0) + prop.confidence - best = max(votes, key=votes.get) - prop.class_name = best - if prop.track_id in self.tracks: - self.tracks[prop.track_id].class_name = best - - for track in self.tracks.values(): - if track.missed_frames == 0 or not track.history: - continue # matched (or spawned) this frame - if not (1 <= track.missed_frames <= self.propagate_max): - continue - real_hits = sum(1 for h in track.history if not h.get("propagated")) - if real_hits < self.min_hits_to_propagate: - continue - px, py = float(track.state[0]), float(track.state[1]) - x0, x1, y0, y1 = self.bounds - if not (x0 <= px < x1 and y0 <= py < y1): - continue # object likely left the field of view: do not hallucinate - last_box = track.history[-1]["bbox_3d"] - bbox = [px, py, last_box[2], last_box[3], last_box[4], last_box[5], last_box[6]] - conf = round(track.confidence * (0.8 ** track.missed_frames), 4) - track.history.append({ - "frame_id": frame_id, "bbox_3d": bbox, - "confidence": conf, "propagated": True, - }) - prop = Object3D(bbox_3d=bbox, class_name=track.class_name, - confidence=conf, track_id=track.track_id) - self.last_propagated.add(track.track_id) - out.append(prop) - return out - - -def tracks_to_dicts(per_frame: Dict[str, List[Dict]]) -> List[Dict]: - """Group per-frame labeled boxes (with track_id) into MOT-shaped tracks.""" - tracks: Dict[object, Dict] = {} - for frame_id in sorted(per_frame.keys()): - for box in per_frame[frame_id]: - tid = box.get("track_id") - if tid is None: - continue - entry = tracks.setdefault(tid, {"track_id": tid, - "class_name": box.get("class_name", "unknown"), - "frames": []}) - entry["frames"].append({"frame_id": frame_id, "bbox_3d": box["bbox_3d"]}) - return list(tracks.values()) diff --git a/hf-space/sensorflow/bevfusion/scenes.py b/hf-space/sensorflow/bevfusion/scenes.py deleted file mode 100644 index 1e10ef7e76e562c8cad04dade78e05e6c1c1fb1c..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/bevfusion/scenes.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Deterministic synthetic multi-sensor scenes with known 3D ground truth. - -Mirrors the conventions of the existing evaluation generator -(sensorflow/evaluation/synthetic.py: ego drives +x at 10 m/s, objects have -world-frame kinematics, class dimensions from CLASS_DIMS) but is purpose-built -for the sensor-simulation study: every ground-truth box carries a per-frame -camera-occlusion flag (planted contiguous occlusion windows) so that both -sensor simulators and the cohort evaluation can reason about occlusion. - -Conditions cycle deterministically across sequences (day/clear, night/clear, -day/rain) so every cohort is well represented at any n_sequences >= 3. -""" - -from __future__ import annotations - -import math -from typing import List - -import numpy as np -from pydantic import BaseModel, Field - -# Reuse the platform's class dimension priors rather than duplicating them. -from sensorflow.evaluation.synthetic import CLASS_DIMS, CLASS_POOL - -CLASSES = list(CLASS_DIMS.keys()) -SAFETY_CLASSES = {"pedestrian", "cyclist", "motorcycle"} - -DT = 0.1 -EGO_SPEED = 10.0 - - -class GTBox(BaseModel): - instance_id: str - class_name: str - bbox_3d: List[float] # [x, y, z, l, w, h, yaw] ego frame - occluded: bool = False # camera line-of-sight blocked this frame - distance: float = 0.0 - - -class SceneFrame(BaseModel): - frame_id: str - index: int - gt: List[GTBox] = Field(default_factory=list) - - -class SceneSequence(BaseModel): - sequence_id: str - time_of_day: str = "day" # day | night - weather: str = "clear" # clear | rain - frames: List[SceneFrame] = Field(default_factory=list) - - -def generate_sequences(n_sequences: int = 6, frames_per_sequence: int = 24, - seed: int = 7) -> List[SceneSequence]: - """Generate deterministic sequences with planted camera-occlusion windows.""" - sequences: List[SceneSequence] = [] - for qi in range(n_sequences): - rng = np.random.default_rng(seed * 100003 + qi * 7919) - time_of_day = "night" if qi % 3 == 1 else "day" - weather = "rain" if qi % 3 == 2 else "clear" - seq = SceneSequence(sequence_id=f"bev-seq-{seed}-{qi}", - time_of_day=time_of_day, weather=weather) - - num_objects = int(rng.integers(7, 11)) - objects = [] - for oi in range(num_objects): - cls = CLASS_POOL[int(rng.integers(0, len(CLASS_POOL)))] - l, w, h = CLASS_DIMS[cls] - # Rejection-sample spawn positions so objects start >= 5 m apart - # (keeps BEV peaks separable; documented decode limitation). - for _ in range(30): - x0 = float(rng.uniform(10, 68)) - y0 = float(rng.uniform(-13, 13)) - if all(math.hypot(x0 - o["x"], y0 - o["y"]) >= 5.0 for o in objects): - break - heading = (float(rng.uniform(-math.pi, math.pi)) if cls == "pedestrian" - else float(rng.choice([0.0, math.pi, math.pi / 2, -math.pi / 2]))) - speed = {"pedestrian": 1.4, "cyclist": 4.5, "motorcycle": 9.0, - "vehicle": 8.0, "truck": 7.0}[cls] * float(rng.uniform(0.6, 1.2)) - objects.append({ - "iid": f"{seq.sequence_id}-obj-{oi}", "cls": cls, "dims": (l, w, h), - "x": x0, "y": y0, - "vx": speed * math.cos(heading), "vy": speed * math.sin(heading), - "yaw": heading, - }) - - # Plant camera-occlusion windows: 2 objects per sequence lose camera - # line of sight for a contiguous 2-3 frame window mid-sequence. - occlusion = {} - if objects and frames_per_sequence >= 10: - occluded_idx = rng.choice(len(objects), size=min(2, len(objects)), replace=False) - for k, oi in enumerate(occluded_idx): - start = int(rng.integers(4, max(5, frames_per_sequence - 6))) - length = int(rng.integers(2, 4)) - occlusion[objects[int(oi)]["iid"]] = (start, start + length) - - for fi in range(frames_per_sequence): - ego_x = EGO_SPEED * DT * fi - frame = SceneFrame(frame_id=f"{seq.sequence_id}-f{fi:04d}", index=fi) - for obj in objects: - wx = obj["x"] + obj["vx"] * DT * fi - wy = obj["y"] + obj["vy"] * DT * fi - rel_x, rel_y = wx - ego_x, wy - if rel_x < 3.0 or rel_x > 78.0 or abs(rel_y) > 22.0: - continue - l, w, h = obj["dims"] - win = occlusion.get(obj["iid"]) - frame.gt.append(GTBox( - instance_id=obj["iid"], - class_name=obj["cls"], - bbox_3d=[round(rel_x, 3), round(rel_y, 3), round(h / 2, 3), - l, w, h, round(obj["yaw"], 4)], - occluded=bool(win and win[0] <= fi < win[1]), - distance=round(math.hypot(rel_x, rel_y), 3), - )) - seq.frames.append(frame) - sequences.append(seq) - return sequences - - -def gt_tracks(sequences: List[SceneSequence]) -> List[dict]: - """Ground-truth tracks in the shape expected by sensorflow.metrics.temporal_mot.""" - by_instance: dict = {} - for seq in sequences: - for frame in seq.frames: - for gt in frame.gt: - entry = by_instance.setdefault( - gt.instance_id, - {"instance_id": gt.instance_id, "track_id": gt.instance_id, - "class_name": gt.class_name, "frames": []}) - entry["frames"].append({"frame_id": frame.frame_id, "bbox_3d": gt.bbox_3d}) - return list(by_instance.values()) diff --git a/hf-space/sensorflow/bevfusion/sensors.py b/hf-space/sensorflow/bevfusion/sensors.py deleted file mode 100644 index 5ad4c99b2afbd7460bb2cc65fda65ad9dd4a3c1f..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/bevfusion/sensors.py +++ /dev/null @@ -1,225 +0,0 @@ -"""Camera and LiDAR detection simulators with modality-specific failure modes. - -These stand in for the learned detection backbones of a real BEVFusion stack. -Detections are sampled from ground truth with honest, physically motivated -error models — the complementarity that the fusion exploits is real: - -Camera ("perception-v1-camera" front end): - - misses under occlusion (line of sight blocked), at night (with a - distance ramp: dark + small pixels), lightly in rain; - - monocular depth ambiguity: range estimate noisy (~7% of range), bearing - precise -> anisotropic BEV covariance elongated along the viewing ray; - - good class discrimination (94% correct, confusions within size groups); - - occasional false positives (glare phantoms, more at night). - -LiDAR: - - accurate range/geometry (~0.1-0.2 m position, ~3.5% dims); - - degraded at long distance (sparse returns beyond ~45 m) and in heavy - rain (spray attenuation growing with distance); mild degradation under - partial occlusion (fewer returns, but the sensor sits higher and sees - in 3D, so much less than the camera); - - weak class discrimination: geometry-template guess with low confidence - (semantics is the camera's job); - - unaffected by night — this is the key complementarity for night recall. - -All randomness is seeded per (seed, sequence, modality): deterministic. -""" - -from __future__ import annotations - -import math -from typing import List, Optional - -import numpy as np -from pydantic import BaseModel, Field - -from sensorflow.bevfusion.geometry import ( - backproject, make_camera_extrinsic, make_intrinsics, project_to_image, - CAM_W, CAM_H, -) -from sensorflow.bevfusion.scenes import CLASS_DIMS, CLASSES, SceneFrame, SceneSequence - -# Class confusion groups (similar appearance / similar footprint). -_CONFUSION = { - "vehicle": ["truck"], - "truck": ["vehicle"], - "pedestrian": ["cyclist"], - "cyclist": ["pedestrian", "motorcycle"], - "motorcycle": ["cyclist"], -} - -DEFAULT_K = make_intrinsics() -DEFAULT_CAM_TO_EGO = make_camera_extrinsic(tx=0.2, ty=0.0, tz=1.5) - - -class Detection(BaseModel): - """A single-modality detection expressed in the ego/BEV frame.""" - - modality: str # "camera" | "lidar" - x: float - y: float - z: float = 0.0 - cov: List[List[float]] = Field(default_factory=lambda: [[0.04, 0.0], [0.0, 0.04]]) - dims: List[float] = Field(default_factory=lambda: [4.5, 1.9, 1.6]) # l, w, h - yaw: float = 0.0 - class_name: str = "vehicle" - class_confidence: float = 0.5 - confidence: float = 0.5 - source_instance_id: Optional[str] = None # debug/tests only; engines never read it - - -def _ray_cov(x: float, y: float, sigma_along: float, sigma_cross: float, - origin_x: float = 0.2, origin_y: float = 0.0) -> List[List[float]]: - """BEV covariance elongated along the sensor->object ray.""" - theta = math.atan2(y - origin_y, x - origin_x) - c, s = math.cos(theta), math.sin(theta) - rot = np.array([[c, -s], [s, c]]) - cov = rot @ np.diag([sigma_along ** 2, sigma_cross ** 2]) @ rot.T - return cov.tolist() - - -# ------------------------------------------------------------------ camera - - -def simulate_camera(frame: SceneFrame, seq: SceneSequence, rng: np.random.Generator, - cam_to_ego: np.ndarray = DEFAULT_CAM_TO_EGO, - K: np.ndarray = DEFAULT_K) -> List[Detection]: - night = seq.time_of_day == "night" - rain = seq.weather == "rain" - dets: List[Detection] = [] - - for gt in frame.gt: - gx, gy, gz, l, w, h, yaw = gt.bbox_3d - d = gt.distance - p_miss = 0.04 + 0.10 * min(1.0, d / 70.0) - if gt.occluded: - p_miss += 0.82 - if night: - p_miss += 0.35 + 0.25 * min(1.0, d / 60.0) - if rain: - p_miss += 0.06 - if rng.random() < min(p_miss, 0.97): - continue - - proj = project_to_image(np.array([gx, gy, gz]), cam_to_ego, K) - if proj is None: - continue - u, v, r_true = proj - u += float(rng.normal(0, 0.4)) - v += float(rng.normal(0, 0.4)) - if not (0 <= u <= CAM_W and 0 <= v <= CAM_H): - continue - - # Monocular depth: multiplicative range noise along the precise ray. - r_est = r_true * float(1.0 + rng.normal(0, 0.07)) - p_ego = backproject(u, v, max(r_est, 1.0), cam_to_ego, K) - dims = [max(0.3, dim * float(1.0 + rng.normal(0, 0.10))) for dim in (l, w, h)] - - if rng.random() < 0.94: - cls = gt.class_name - else: - cls = str(rng.choice(_CONFUSION[gt.class_name])) - sigma_along = max(0.35, 0.07 * d) - sigma_cross = max(0.12, 0.004 * d) - dets.append(Detection( - modality="camera", - x=float(p_ego[0]), y=float(p_ego[1]), z=dims[2] / 2, # ground-plane assumption - cov=_ray_cov(float(p_ego[0]), float(p_ego[1]), sigma_along, sigma_cross), - dims=dims, - yaw=float(yaw + rng.normal(0, math.radians(8))), - class_name=cls, - class_confidence=float(np.clip(rng.normal(0.88, 0.05), 0.5, 0.99)), - confidence=float(np.clip(0.88 - d / 250.0 + rng.normal(0, 0.05), 0.30, 0.99)), - source_instance_id=gt.instance_id, - )) - - # False positives: glare/texture phantoms, more frequent at night. - if rng.random() < (0.10 if night else 0.05): - fx = float(rng.uniform(8, 70)) - fy = float(rng.uniform(-15, 15)) - cls = str(rng.choice(CLASSES)) - l, w, h = CLASS_DIMS[cls] - d = math.hypot(fx, fy) - dets.append(Detection( - modality="camera", x=fx, y=fy, z=h / 2, - cov=_ray_cov(fx, fy, max(0.35, 0.07 * d), max(0.12, 0.004 * d)), - dims=[l, w, h], yaw=float(rng.uniform(-math.pi, math.pi)), - class_name=cls, class_confidence=0.55, - confidence=float(rng.uniform(0.35, 0.55)), - )) - return dets - - -# ------------------------------------------------------------------ lidar - - -def _template_class(dims: List[float], rng: np.random.Generator) -> str: - """Weak geometric classification: nearest size template, often confused. - - Real LiDAR-only detectors get geometry right but semantics wrong; we model - that as a 55% template guess / 45% confusion-group draw. - """ - best, best_err = "vehicle", float("inf") - for cls, (tl, tw, th) in CLASS_DIMS.items(): - err = (math.log(dims[0] / tl) ** 2 + math.log(dims[1] / tw) ** 2 - + math.log(dims[2] / th) ** 2) - if err < best_err: - best, best_err = cls, err - if rng.random() < 0.55: - return best - return str(rng.choice(_CONFUSION[best] + [best])) - - -def simulate_lidar(frame: SceneFrame, seq: SceneSequence, - rng: np.random.Generator) -> List[Detection]: - rain = seq.weather == "rain" - dets: List[Detection] = [] - - for gt in frame.gt: - gx, gy, gz, l, w, h, yaw = gt.bbox_3d - d = gt.distance - p_miss = 0.02 + 0.60 * max(0.0, (d - 45.0) / 35.0) - if rain: - p_miss += 0.03 + 0.25 * max(0.0, (d - 25.0) / 50.0) - if gt.occluded: - p_miss += 0.15 # partial returns: far milder than the camera's 0.82 - if rng.random() < min(p_miss, 0.97): - continue - - sigma = 0.10 + 0.0015 * d - dims = [max(0.3, dim * float(1.0 + rng.normal(0, 0.035))) for dim in (l, w, h)] - dets.append(Detection( - modality="lidar", - x=float(gx + rng.normal(0, sigma)), - y=float(gy + rng.normal(0, sigma)), - z=float(gz + rng.normal(0, sigma)), - cov=[[sigma ** 2, 0.0], [0.0, sigma ** 2]], - dims=dims, - yaw=float(yaw + rng.normal(0, math.radians(3))), - class_name=_template_class(dims, rng), - class_confidence=float(np.clip(rng.normal(0.40, 0.05), 0.2, 0.6)), - confidence=float(np.clip(0.93 - 0.5 * max(0.0, (d - 40.0) / 40.0) - - (0.08 if rain else 0.0) + rng.normal(0, 0.03), - 0.30, 0.99)), - source_instance_id=gt.instance_id, - )) - - # Rain clutter (spray) false positives: weak, usually below decode threshold. - if rain and rng.random() < 0.06: - fx, fy = float(rng.uniform(5, 40)), float(rng.uniform(-12, 12)) - dets.append(Detection( - modality="lidar", x=fx, y=fy, z=0.5, - cov=[[0.04, 0.0], [0.0, 0.04]], - dims=[0.8, 0.8, 1.0], yaw=0.0, - class_name="pedestrian", class_confidence=0.25, - confidence=float(rng.uniform(0.30, 0.45)), - )) - return dets - - -def camera_rng(seed: int, seq_index: int) -> np.random.Generator: - return np.random.default_rng(seed * 100003 + seq_index * 7919 + 1) - - -def lidar_rng(seed: int, seq_index: int) -> np.random.Generator: - return np.random.default_rng(seed * 100003 + seq_index * 7919 + 2) diff --git a/hf-space/sensorflow/cli/__init__.py b/hf-space/sensorflow/cli/__init__.py deleted file mode 100644 index 33166fc3f17741dfd468668f8cafaa4951db0c18..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/cli/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CLI entrypoints for the 3D perception pipeline.""" diff --git a/hf-space/sensorflow/cli/__main__.py b/hf-space/sensorflow/cli/__main__.py deleted file mode 100644 index 82e388bf47a71228dc72e474db931486200cff86..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/cli/__main__.py +++ /dev/null @@ -1 +0,0 @@ -"""Make sensorflow.cli modules runnable via python -m.""" diff --git a/hf-space/sensorflow/cli/auto_label.py b/hf-space/sensorflow/cli/auto_label.py deleted file mode 100644 index 6e870a076e8aad7142855324ce30b5e3efbc62b3..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/cli/auto_label.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -"""Run SAM-based 3D auto-labeling on a fused sequence.""" - -import argparse -from pathlib import Path - -from sensorflow.perception_automator import PerceptionAutomator -from sensorflow.schemas.unified_frame import UnifiedSequence - - -def main(): - parser = argparse.ArgumentParser(description="3D auto-labeling with SAM") - parser.add_argument("--manifest", required=True) - parser.add_argument("--sam-checkpoint", default="models/sam_vit_b.pth") - parser.add_argument("--device", default="cpu") - parser.add_argument("--no-sam", action="store_true") - args = parser.parse_args() - - manifest_path = Path(args.manifest) - sequence = UnifiedSequence.load(manifest_path) - output_dir = manifest_path.parent / "proposals" - - automator = PerceptionAutomator( - sam_checkpoint=args.sam_checkpoint, - device=args.device, - use_sam=not args.no_sam, - ) - proposals = automator.run_sequence(sequence, output_dir) - print(f"Generated proposals for {len(proposals)} frames -> {output_dir}") - - -if __name__ == "__main__": - main() diff --git a/hf-space/sensorflow/cli/benchmark.py b/hf-space/sensorflow/cli/benchmark.py deleted file mode 100644 index 934e9d7cade778e250f742ed204224c125e5eb41..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/cli/benchmark.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -"""Run quality gate benchmarking.""" - -import argparse -import json -from pathlib import Path - -from sensorflow.quality_gate import QualityGate -from sensorflow.schemas.unified_frame import UnifiedSequence -from sensorflow.mitl_copilot import MitlCopilot - - -def main(): - parser = argparse.ArgumentParser(description="Quality gate benchmark") - parser.add_argument("--manifest", required=True) - parser.add_argument("--tracks", default=None) - args = parser.parse_args() - - manifest_path = Path(args.manifest) - sequence = UnifiedSequence.load(manifest_path) - tracks_path = Path(args.tracks) if args.tracks else manifest_path.parent / "tracks.json" - - with open(tracks_path) as f: - pred_tracks = json.load(f) - - gate = QualityGate() - results = gate.evaluate(sequence, pred_tracks) - out_dir = gate.save_results(sequence.sequence_id, results) - - copilot = MitlCopilot() - if not results["passed"]: - copilot.route_edge_cases( - sequence.sequence_id, - results["metric_card"], - pred_tracks, - ) - - print(f"Quality gate {'PASSED' if results['passed'] else 'FAILED'} -> {out_dir}") - print(json.dumps(results["metric_card"], indent=2)) - - -if __name__ == "__main__": - main() diff --git a/hf-space/sensorflow/cli/ingest.py b/hf-space/sensorflow/cli/ingest.py deleted file mode 100644 index a6a7d4a2fa1a3a4d50706c09b73c5c43d72a7ac8..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/cli/ingest.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -"""Ingest and fuse multi-vendor sensor data.""" - -import argparse -import json -from pathlib import Path - -from sensorflow.dataset_fusion_engine import DatasetFusionEngine - - -def main(): - parser = argparse.ArgumentParser(description="Ingest and fuse sensor data") - parser.add_argument("--vendors", nargs="+", default=["alpamayo", "waymo"]) - parser.add_argument("--sequence-id", default="seq_001") - parser.add_argument("--alpamayo-source", default=None) - parser.add_argument("--waymo-source", default=None) - args = parser.parse_args() - - engine = DatasetFusionEngine() - alp_src = json.loads(args.alpamayo_source) if args.alpamayo_source else None - way_src = json.loads(args.waymo_source) if args.waymo_source else None - - sequence = engine.ingest(args.vendors, args.sequence_id, alp_src, way_src) - manifest_path = engine.save_manifest(sequence) - print(f"Ingested {len(sequence.frames)} frames -> {manifest_path}") - - -if __name__ == "__main__": - main() diff --git a/hf-space/sensorflow/cli/track.py b/hf-space/sensorflow/cli/track.py deleted file mode 100644 index bcab6daa49fe72b0f292aebc81d204570fa1e9ca..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/cli/track.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -"""Run temporal tracking on 3D proposals.""" - -import argparse -from pathlib import Path - -from sensorflow.temporal_tracker import TemporalTracker - - -def main(): - parser = argparse.ArgumentParser(description="Temporal 3D tracking") - parser.add_argument("--proposals-dir", required=True) - parser.add_argument("--output", default=None) - args = parser.parse_args() - - proposals_dir = Path(args.proposals_dir) - output_path = Path(args.output) if args.output else proposals_dir.parent / "tracks.json" - - tracker = TemporalTracker() - proposals = TemporalTracker.load_proposals(proposals_dir) - tracks = tracker.run_sequence(proposals, output_path) - print(f"Tracked {len(tracks)} objects -> {output_path}") - - -if __name__ == "__main__": - main() diff --git a/hf-space/sensorflow/dataset_fusion_engine.py b/hf-space/sensorflow/dataset_fusion_engine.py deleted file mode 100644 index 2fbcee395cd9101db9cb7b7f5739be0e8605f6db..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/dataset_fusion_engine.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Dataset fusion engine: temporal sync, stratification, manifest output.""" - -from __future__ import annotations - -import json -from collections import Counter -from pathlib import Path -from typing import Any, Dict, List, Optional - -from sensorflow.adapters.alpamayo_adapter import AlpamayoAdapter, DEFAULT_ALPAMAYO_SAMPLES -from sensorflow.adapters.waymo_adapter import WaymoAdapter -from sensorflow.schemas.taxonomy_axes import assign_taxonomy_axes -from sensorflow.schemas.unified_frame import FusedFrame, UnifiedSequence - - -class DatasetFusionEngine: - """Ingest, fuse, and stratify multi-vendor sensor sequences.""" - - def __init__(self, sync_tolerance_us: int = 50_000): - self.sync_tolerance_us = sync_tolerance_us - self.alpamayo = AlpamayoAdapter() - self.waymo = WaymoAdapter() - - def ingest( - self, - vendors: List[str], - sequence_id: str, - alpamayo_source: Optional[Dict[str, Any]] = None, - waymo_source: Optional[Dict[str, Any]] = None, - ) -> UnifiedSequence: - sequences: List[UnifiedSequence] = [] - - if "alpamayo" in vendors: - src = alpamayo_source or DEFAULT_ALPAMAYO_SAMPLES.get("physical_ai", {}) - sequences.append(self.alpamayo.load(src, f"{sequence_id}_alp")) - - if "waymo" in vendors: - sequences.append(self.waymo.load(waymo_source or {}, f"{sequence_id}_way")) - - if not sequences: - sequences.append(self.alpamayo.load(DEFAULT_ALPAMAYO_SAMPLES["physical_ai"], sequence_id)) - - fused = self._merge_sequences(sequences, sequence_id) - fused = self._stratify(fused) - return fused - - def _merge_sequences(self, sequences: List[UnifiedSequence], sequence_id: str) -> UnifiedSequence: - if len(sequences) == 1: - seq = sequences[0] - seq.sequence_id = sequence_id - return seq - - all_frames: List[FusedFrame] = [] - for seq in sequences: - all_frames.extend(seq.frames) - all_frames.sort(key=lambda f: f.timestamp_us) - - vendor = "mixed" if len(sequences) > 1 else sequences[0].vendor - return UnifiedSequence( - sequence_id=sequence_id, - vendor=vendor, - frames=all_frames, - calibration=sequences[0].calibration, - taxonomy_manifest={"merged_vendors": [s.vendor for s in sequences]}, - ) - - def _stratify(self, sequence: UnifiedSequence) -> UnifiedSequence: - """Tag frames/objects with six-axis taxonomy and build stratification manifest.""" - strata: Counter = Counter() - for frame in sequence.frames: - speed = frame.ego_pose.speed_kmh - for gt in frame.ground_truth: - gt.taxonomy_axes = assign_taxonomy_axes(gt.class_name, speed_kmh=speed) - key = f"{gt.taxonomy_axes.mode}|{gt.taxonomy_axes.actor}|{gt.taxonomy_axes.infra}" - strata[key] += 1 - sequence.taxonomy_manifest["stratification"] = dict(strata) - sequence.taxonomy_manifest["total_frames"] = len(sequence.frames) - sequence.taxonomy_manifest["total_objects"] = sum(len(f.ground_truth) for f in sequence.frames) - return sequence - - def save_manifest(self, sequence: UnifiedSequence, output_dir: Optional[Path] = None) -> Path: - output_dir = output_dir or Path("runs/pipeline") / sequence.sequence_id - manifest_path = output_dir / "manifest.json" - sequence.save(manifest_path) - state_path = Path("runs/pipeline/state.json") - state = self._load_state(state_path) - state[sequence.sequence_id] = { - "ingest_complete": True, - "frames": len(sequence.frames), - "vendor": sequence.vendor, - "manifest": str(manifest_path), - } - state_path.parent.mkdir(parents=True, exist_ok=True) - with open(state_path, "w") as f: - json.dump(state, f, indent=2) - return manifest_path - - @staticmethod - def _load_state(path: Path) -> Dict[str, Any]: - if path.exists(): - with open(path) as f: - return json.load(f) - return {} - - def get_status(self, sequence_id: str) -> Dict[str, Any]: - state = self._load_state(Path("runs/pipeline/state.json")) - return state.get(sequence_id, {"ingest_complete": False}) diff --git a/hf-space/sensorflow/evaluation/__init__.py b/hf-space/sensorflow/evaluation/__init__.py deleted file mode 100644 index 6c4aacf9a0016881620ddc46140036d2949a8a67..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""L4 Perception Label Evaluation, Rare Event Detection & Automated Triage engine.""" diff --git a/hf-space/sensorflow/evaluation/api.py b/hf-space/sensorflow/evaluation/api.py deleted file mode 100644 index 3bb957ee952872eedec0481539886d65203f924b..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/api.py +++ /dev/null @@ -1,594 +0,0 @@ -"""FastAPI router for the label evaluation platform (included by app_backend).""" - -from __future__ import annotations - -import asyncio -import json -from datetime import datetime, timezone -from typing import Dict, List, Optional - -from fastapi import APIRouter, HTTPException -from fastapi.responses import StreamingResponse -from pydantic import BaseModel, Field - -from sensorflow.evaluation import copilot, reporting, synthetic -from sensorflow.evaluation.pipeline import get_pipeline -from sensorflow.evaluation.process_units import usage_summary -from sensorflow.evaluation.records import RelabelingAction, get_store -from sensorflow.evaluation.triage import QualityPolicy, load_policy, save_policy - -router = APIRouter(prefix="/api") - - -# ------------------------------------------------------------------ request bodies - - -class GenerateDatasetRequest(BaseModel): - name: str = "synthetic-perception" - num_sequences: int = 6 - frames_per_sequence: int = 30 - seed: int = 7 - - -class RunPipelineRequest(BaseModel): - dataset_id: str - policy_id: Optional[str] = None - - -class PrecheckRequest(BaseModel): - dataset_id: Optional[str] = None - - -class ReviewActionRequest(BaseModel): - action: str - corrected_bbox_3d: Optional[List[float]] = None - corrected_class: Optional[str] = None - merge_with_track_id: Optional[str] = None - note: Optional[str] = None - - -class TrainRequest(BaseModel): - dataset_version: str - model_version: Optional[str] = None - configuration: Dict = Field(default_factory=dict) - quality_policy: Optional[str] = None - training_parameters: Dict = Field(default_factory=dict) - - -class BenchmarkTechniquesRequest(BaseModel): - dataset_id: Optional[str] = None - - -class CopilotRequest(BaseModel): - context_type: str = "general" - annotation_id: Optional[str] = None - event_id: Optional[str] = None - model_version: Optional[str] = None - extra: Dict = Field(default_factory=dict) - - -# ------------------------------------------------------------------ serializers - - -def _dataset_summary(d) -> Dict: - store = get_store() - anns = store.where("annotations", dataset_id=d.dataset_id) - verified = sum(1 for a in anns if a.status == "VERIFIED") - coverage = d.gt_coverage - conf = "none" - if coverage > 0: - conf = "high" if d.gt_type in ("HUMAN_VERIFIED_GROUND_TRUTH", "GOLD_STANDARD") else ( - "medium" if d.gt_type == "VENDOR_GROUND_TRUTH" else "low") - return { - "dataset_id": d.dataset_id, - "name": d.name, - "version": d.version, - "created_at": d.created_at, - "num_scenes": d.num_scenes, - "num_sequences": d.num_sequences, - "num_frames": d.num_frames, - "num_annotations": len(anns) or d.num_annotations, - "gt_availability": { - "has_reference": coverage > 0, - "gt_type": d.gt_type if coverage > 0 else None, - "coverage": coverage, - "evaluation_confidence": conf, - }, - "lineage": d.lineage.model_dump(), - "status": d.status, - "verified_count": verified, - } - - -def _pipeline_state() -> Dict: - pipe = get_pipeline() - store = get_store() - open_tasks = len([t for t in store.all("review_tasks") if t.status != "resolved"]) - return { - "running": pipe.running, - "stage": pipe.stage, - "services": pipe.service_status(), - "queue": pipe.queue.stats(), - "counters": reporting.counters(store, pipe.active_dataset), - "review_queue_count": open_tasks, - "regression_alert": pipe.regression_alert, - "last_run_id": pipe.last_run_id, - } - - -def _active_training_job() -> Optional[Dict]: - store = get_store() - jobs = sorted(store.all("training_jobs"), key=lambda j: j.started_at) - if not jobs: - return None - running = [j for j in jobs if j.status == "running"] - job = running[-1] if running else jobs[-1] - d = job.model_dump() - d["logs"] = d["logs"][-50:] - return d - - -# ------------------------------------------------------------------ datasets - - -@router.get("/datasets") -def list_datasets(): - store = get_store() - ds = sorted(store.all("datasets"), key=lambda d: d.created_at) - return {"datasets": [_dataset_summary(d) for d in ds]} - - -@router.get("/datasets/{dataset_id}") -def get_dataset(dataset_id: str): - store = get_store() - d = store.get("datasets", dataset_id) - if d is None: - raise HTTPException(404, f"Unknown dataset {dataset_id}") - return _dataset_summary(d) - - -@router.post("/dataset/precheck") -def dataset_precheck(req: PrecheckRequest): - store = get_store() - checks = [] - dataset = None - if req.dataset_id: - dataset = store.get("datasets", req.dataset_id) - if dataset is None: - raise HTTPException(404, f"Unknown dataset {req.dataset_id}") - else: - ds = sorted(store.all("datasets"), key=lambda d: d.created_at) - dataset = ds[-1] if ds else None - - def check(gate, actual, threshold, passed, applicable=True): - checks.append({"gate": gate, "actual": actual, "threshold": threshold, - "passed": passed, "applicable": applicable}) - - if dataset is None: - check("dataset_exists", False, True, False) - return {"status": "failed", "message": "No dataset available — generate one first.", "checks": checks} - - frames = store.where("frames", dataset_id=dataset.dataset_id) - check("dataset_exists", True, True, True) - check("frames_present", len(frames), ">= 10", len(frames) >= 10) - seqs = store.where("sequences", dataset_id=dataset.dataset_id) - check("sequences_present", len(seqs), ">= 1", len(seqs) >= 1) - with_gt = sum(1 for f in frames if f.gt_boxes) - check("reference_gt_coverage", f"{with_gt}/{len(frames)} frames", "> 0", - with_gt > 0) - check("gt_type_labeled", dataset.gt_type or "none", "explicit GT type", dataset.gt_type is not None) - check("lidar_points_expected", frames[0].num_lidar_points if frames else 0, "> 0", - bool(frames and frames[0].num_lidar_points > 0)) - policy = load_policy() - check("quality_policy_loaded", policy.policy_id, "loadable", True) - passed = all(c["passed"] for c in checks if c["applicable"]) - return { - "status": "success" if passed else "warning", - "message": ("Dataset ready for evaluation pipeline." - if passed else "Precheck found issues; review the failed gates."), - "checks": checks, - "dataset_id": dataset.dataset_id, - } - - -@router.post("/labeleval/datasets/generate") -def generate_dataset(req: GenerateDatasetRequest): - store = get_store() - dataset = synthetic.generate_dataset( - store, name=req.name, num_sequences=req.num_sequences, - frames_per_sequence=req.frames_per_sequence, seed=req.seed, - version=f"v{len(store.all('datasets')) + 1}", - ) - pipe = get_pipeline() - pipe.active_dataset = dataset.dataset_id - store.save() - return _dataset_summary(dataset) - - -@router.post("/labeleval/run") -def run_pipeline(req: RunPipelineRequest): - pipe = get_pipeline() - store = get_store() - if store.get("datasets", req.dataset_id) is None: - raise HTTPException(404, f"Unknown dataset {req.dataset_id}") - try: - run_id = pipe.run(req.dataset_id, background=True) - except RuntimeError as e: - raise HTTPException(409, str(e)) - return {"run_id": run_id, "status": "started"} - - -# ------------------------------------------------------------------ pipeline / queue / overview - - -@router.get("/labeleval/pipeline") -def pipeline_state(): - return _pipeline_state() - - -@router.get("/queue/status") -def queue_status(): - return get_pipeline().queue.stats() - - -@router.get("/labeleval/overview") -def overview(): - return reporting.overview(get_store(), get_pipeline()) - - -@router.get("/labeleval/funnel") -def funnel(): - return reporting.funnel(get_store(), get_pipeline().active_dataset) - - -# ------------------------------------------------------------------ frames / evaluations - - -@router.get("/labeleval/frames") -def list_frames(dataset_id: Optional[str] = None): - store = get_store() - did = dataset_id or get_pipeline().active_dataset - frames = sorted(store.where("frames", dataset_id=did), key=lambda f: f.index) if did else [] - return {"frame_ids": [f.frame_id for f in frames]} - - -@router.get("/labeleval/frames/{frame_id}") -def get_frame(frame_id: str): - store = get_store() - summary = reporting.frame_summary(store, frame_id) - if summary is None: - raise HTTPException(404, f"Unknown frame {frame_id}") - frame = store.get("frames", frame_id) - seq = store.get("sequences", frame.sequence_id) - prev_id = next_id = None - if seq: - idx = seq.frame_ids.index(frame_id) if frame_id in seq.frame_ids else -1 - if idx > 0: - prev_id = seq.frame_ids[idx - 1] - if 0 <= idx < len(seq.frame_ids) - 1: - next_id = seq.frame_ids[idx + 1] - return {"frame": summary, "prev": prev_id, "next": next_id} - - -@router.get("/labeleval/evaluations/{annotation_id}") -def get_evaluation(annotation_id: str): - rec = reporting.evaluation_record(get_store(), annotation_id) - if rec is None: - raise HTTPException(404, f"Unknown annotation {annotation_id}") - return rec - - -@router.get("/labeleval/evaluations") -def list_evaluations(dataset_id: Optional[str] = None, limit: int = 200, offset: int = 0): - store = get_store() - did = dataset_id or get_pipeline().active_dataset - anns = sorted(store.where("annotations", dataset_id=did), key=lambda a: a.annotation_id) if did else [] - total = len(anns) - out = [] - for a in anns[offset:offset + limit]: - rec = reporting.evaluation_record(store, a.annotation_id) - if rec: - out.append(rec) - return {"total": total, "records": out} - - -# ------------------------------------------------------------------ haystack / rare events - - -@router.get("/labeleval/haystack") -def haystack(dataset_id: Optional[str] = None): - did = dataset_id or get_pipeline().active_dataset - return {"points": reporting.haystack(get_store(), did)} - - -@router.get("/labeleval/rare-events") -def rare_events(): - store = get_store() - did = get_pipeline().active_dataset - events = store.where("rare_events", dataset_id=did) if did else store.all("rare_events") - events = sorted(events, key=lambda e: (-{"critical": 3, "high": 2, "medium": 1, "low": 0}[e.severity], -e.rarity_score)) - return {"events": [e.model_dump() for e in events]} - - -@router.get("/labeleval/rare-events/{event_id}") -def rare_event(event_id: str): - e = get_store().get("rare_events", event_id) - if e is None: - raise HTTPException(404, f"Unknown rare event {event_id}") - return e.model_dump() - - -# ------------------------------------------------------------------ quality - - -@router.get("/quality/metrics") -def quality_metrics(dataset_id: Optional[str] = None): - did = dataset_id or get_pipeline().active_dataset - return reporting.quality_metrics(get_store(), did) - - -@router.get("/quality/groups") -def quality_groups(dataset_id: Optional[str] = None): - did = dataset_id or get_pipeline().active_dataset - if did is None: - return {"dataset_id": None, "total": 0, "verification_rate": 0.0, "groups": []} - return reporting.quality_groups(get_store(), did) - - -@router.get("/quality/groups/{group_id}") -def quality_group_detail(group_id: str): - detail = reporting.group_detail(get_store(), group_id) - if detail is None: - raise HTTPException(404, f"Unknown group {group_id}") - return detail - - -@router.get("/labeleval/policy") -def get_policy(): - return load_policy().model_dump() - - -@router.post("/labeleval/policy") -def set_policy(policy: QualityPolicy): - save_policy(policy) - get_store().audit("policy_updated", "QualityPolicy", policy.policy_id, "thresholds updated", "operator") - return policy.model_dump() - - -# ------------------------------------------------------------------ regression - - -@router.get("/regression") -def regression(): - store = get_store() - entries = sorted(store.all("regressions"), key=lambda r: r.date, reverse=True) - return { - "entries": [e.model_dump() for e in entries], - "current_alert": get_pipeline().regression_alert, - } - - -# ------------------------------------------------------------------ review / HITL - - -@router.get("/review/tasks") -def review_tasks(status: Optional[str] = None): - store = get_store() - tasks = sorted(store.all("review_tasks"), key=lambda t: t.created_at, reverse=True) - if status: - tasks = [t for t in tasks if t.status == status] - return {"tasks": [_task_payload(t, with_evidence=False) for t in tasks[:300]]} - - -def _task_payload(t, with_evidence: bool = True) -> Dict: - d = t.model_dump() - d["evidence"] = reporting.evaluation_record(get_store(), t.annotation_id) if with_evidence else None - return d - - -@router.get("/review/tasks/{task_id}") -def review_task(task_id: str): - t = get_store().get("review_tasks", task_id) - if t is None: - raise HTTPException(404, f"Unknown review task {task_id}") - return _task_payload(t) - - -@router.post("/review/tasks/{task_id}") -def act_on_review_task(task_id: str, req: ReviewActionRequest): - valid = {"approve", "reject", "correct", "merge_tracks", "split_track", "mark_ignore"} - if req.action not in valid: - raise HTTPException(400, f"Invalid action {req.action}; expected one of {sorted(valid)}") - pipe = get_pipeline() - try: - result = pipe.apply_review(task_id, RelabelingAction(**req.model_dump())) - except KeyError as e: - raise HTTPException(404, str(e)) - task = result["task"] - return { - "task": _task_payload(task), - "revalidation": reporting.evaluation_record(get_store(), task.annotation_id), - "message": result["message"], - } - - -# ------------------------------------------------------------------ models / training - - -@router.get("/models") -def list_models(): - store = get_store() - models = sorted(store.all("models"), key=lambda m: m.created_at) - out = [m.model_dump() for m in models] - if not out: - out = [{ - "model_id": "mdl-bootstrap", - "model_version": "model-v1", - "name": "synthlab-detector", - "trained_on_dataset": None, - "created_at": datetime.now(timezone.utc).isoformat(), - "status": "ready", - "metrics": {"precision": None, "recall": None, "f1": None, "map_3d": None, - "safety_critical_recall": None, "rare_recall": None}, - "regression_status": "baseline", - }] - return {"models": out} - - -@router.post("/train") -def start_training(req: TrainRequest): - pipe = get_pipeline() - try: - job = pipe.start_training( - dataset_version=req.dataset_version, - model_version=req.model_version, - configuration=req.configuration, - quality_policy=req.quality_policy or "", - training_parameters=req.training_parameters, - ) - except KeyError as e: - raise HTTPException(404, str(e)) - except ValueError as e: - raise HTTPException(400, str(e)) - return {"job_id": job.job_id, "model_id": job.model_id, - "model_version": job.model_version, "status": job.status} - - -@router.get("/train/jobs") -def training_jobs(): - store = get_store() - jobs = sorted(store.all("training_jobs"), key=lambda j: j.started_at, reverse=True) - out = [] - for j in jobs: - d = j.model_dump() - d["logs"] = d["logs"][-50:] - out.append(d) - return {"jobs": out} - - -@router.get("/train/jobs/{job_id}") -def training_job(job_id: str): - j = get_store().get("training_jobs", job_id) - if j is None: - raise HTTPException(404, f"Unknown training job {job_id}") - return j.model_dump() - - -# ------------------------------------------------------------------ anomaly config / benchmark - - -@router.get("/labeleval/config") -def get_anomaly_config(): - return get_pipeline().anomaly_config - - -@router.post("/labeleval/config") -def set_anomaly_config(config: Dict): - pipe = get_pipeline() - pipe.anomaly_config = config - store = get_store() - store.meta["anomaly_config"] = config - store.audit("anomaly_config_updated", "AnomalyConfig", "singleton", - f"strategy={config.get('advanced', {}).get('ensemble_strategy')}", "operator") - store.save() - return pipe.anomaly_config - - -@router.post("/benchmark/techniques") -def run_benchmark_techniques(req: BenchmarkTechniquesRequest): - store = get_store() - did = req.dataset_id or get_pipeline().active_dataset - try: - bench = reporting.benchmark_techniques(store, did, get_pipeline().anomaly_config) - except ValueError as e: - raise HTTPException(400, str(e)) - store.save() - return _bench_payload(bench) - - -@router.get("/benchmark/techniques") -def latest_benchmark(): - store = get_store() - benches = sorted(store.all("benchmarks"), key=lambda b: b.created_at) - if not benches: - return {"benchmark_id": "", "rows": [], "highlights": {}, "created_at": ""} - return _bench_payload(benches[-1]) - - -def _bench_payload(bench) -> Dict: - return { - "benchmark_id": bench.benchmark_id, - "rows": [r.model_dump() for r in bench.rows], - "highlights": bench.highlights, - "created_at": bench.created_at, - } - - -# ------------------------------------------------------------------ process units / alerts / audit - - -@router.get("/labeleval/process-units") -def process_units(): - return usage_summary(get_store()) - - -@router.get("/labeleval/alerts") -def alerts(): - store = get_store() - out = sorted(store.all("alerts"), key=lambda a: a.created_at, reverse=True) - return {"alerts": [{ - "alert_id": a.alert_id, - "kind": a.kind, - "severity": a.severity, - "message": a.message, - "evidence_link": {"page": a.evidence_page, "id": a.evidence_id}, - "created_at": a.created_at, - } for a in out]} - - -@router.get("/labeleval/audit") -def audit(limit: int = 200): - store = get_store() - events = sorted(store.all("audit_events"), key=lambda e: e.timestamp, reverse=True) - return {"events": [e.model_dump() for e in events[:limit]]} - - -# ------------------------------------------------------------------ copilot (evidence-based) - - -@router.post("/labeleval/copilot/explain") -def copilot_explain_evidence(req: CopilotRequest): - return copilot.explain(get_store(), req.model_dump()) - - -# ------------------------------------------------------------------ SSE stream - - -def _megaeval_status() -> Dict: - try: - from sensorflow.megaeval.runs import get_mega_store - mstore = get_mega_store() - active = [r.progress_dict() for r in mstore.runs.values() - if r.status in ("queued", "running", "reducing", "materializing")] - return {"active_runs": active, "cache": mstore.router.cache.stats(), - "total_runs": len(mstore.runs)} - except Exception: - return {"active_runs": [], "cache": None, "total_runs": 0} - - -@router.get("/events/stream") -async def events_stream(ticks: int = 600): - async def gen(): - for _ in range(max(1, min(ticks, 3600))): # bounded; client reconnects - payload = { - "ts": datetime.now(timezone.utc).isoformat(), - "pipeline": _pipeline_state(), - "training": _active_training_job(), - "alerts_count": len(get_store().all("alerts")), - "megaeval": _megaeval_status(), - } - yield f"data: {json.dumps(payload)}\n\n" - await asyncio.sleep(1.0) - - return StreamingResponse(gen(), media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) diff --git a/hf-space/sensorflow/evaluation/copilot.py b/hf-space/sensorflow/evaluation/copilot.py deleted file mode 100644 index af24e06ce99dc5636bdce6d71a5943bb7e57bbaa..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/copilot.py +++ /dev/null @@ -1,206 +0,0 @@ -"""MITL Copilot: advisory analysis of evaluation failures. - -Routes to local Ollama/Gemma when reachable; otherwise produces a deterministic -offline analysis derived from the actual evaluation evidence. The Copilot is -advisory only: it never changes metrics, approves/rejects labels, or overrides -gates (spec §35). -""" - -from __future__ import annotations - -import json -from typing import Dict, List, Optional - -import httpx - -from sensorflow.evaluation import reporting -from sensorflow.evaluation.records import EvalStore - -OLLAMA_ENDPOINTS = [ - {"url": "http://localhost:11434/api/chat", "model": "gemma3:latest"}, - {"url": "http://dgx-spark.tail16d8d9.ts.net:11434/api/chat", "model": "gemma4:26b"}, -] - -REASON_HINTS = { - "LOW_IOU": ("Geometric mismatch vs reference", "box regression head under-fitting for this class/range"), - "POSITION_ERROR": ("Center offset beyond tolerance", "LiDAR-camera extrinsic drift or late fusion latency"), - "ORIENTATION_ERROR": ("Heading estimate off", "symmetric object ambiguity / yaw flipping in the detector"), - "INSUFFICIENT_POINT_SUPPORT": ("Too few LiDAR returns inside box", "occlusion, range sparsity, or hallucinated detection"), - "ANOMALY": ("Statistical outlier vs population", "feature combination rarely seen in training distribution"), - "GRADER_DISAGREEMENT": ("Independent graders disagree", "genuinely ambiguous object class or degraded viewpoint"), - "TRACK_FRAGMENTATION": ("Track split into fragments", "missed detections breaking association continuity"), - "ID_SWITCH": ("Identity switched mid-track", "crossing objects with similar appearance/kinematics"), - "MODEL_REGRESSION": ("Model version regressed", "training data shift or a bad checkpoint promotion"), - "LOW_CONFIDENCE": ("Detector confidence very low", "borderline detection near the score threshold"), - "SENSOR_DISAGREEMENT": ("Camera and LiDAR disagree", "single-sensor failure mode (glare, rain scatter, dropout)"), -} - - -def _gather_context(store: EvalStore, req: Dict) -> Dict: - ctx: Dict = {"context_type": req.get("context_type", "general")} - aid = req.get("annotation_id") - if aid: - rec = reporting.evaluation_record(store, aid) - if rec: - ctx["evaluation_record"] = rec - eid = req.get("event_id") - if eid: - ev = store.get("rare_events", eid) - if ev: - ctx["rare_event"] = ev.model_dump() - if req.get("context_type") == "regression" or req.get("model_version"): - regs = sorted(store.all("regressions"), key=lambda r: r.date) - if req.get("model_version"): - regs = [r for r in regs if r.model_version == req["model_version"]] or regs - if regs: - ctx["regression"] = regs[-1].model_dump() - return ctx - - -def _offline_analysis(ctx: Dict) -> Dict: - """Deterministic rule-based analysis assembled from real evidence.""" - ctype = ctx.get("context_type", "general") - evidence: List[str] = [] - factors: List[str] = [] - investigation: List[str] = [] - classification = ctype.replace("_", " ").title() - likely_cause = "Insufficient evidence gathered for a specific cause." - confidence = 0.5 - - rec = ctx.get("evaluation_record") - if rec: - geo = rec["geometry"] - an = rec["anomaly"] - gr = rec["grading"] - dec = rec.get("decision") or {} - reasons = dec.get("failure_reasons", []) - if reasons: - classification = reasons[0].replace("_", " ").title() - if geo.get("iou_3d") is not None: - evidence.append(f"3D IoU vs reference = {geo['iou_3d']:.3f}") - if geo.get("position_error") is not None: - evidence.append(f"Position error = {geo['position_error']:.2f} m") - if geo.get("orientation_error_deg") is not None: - evidence.append(f"Orientation error = {geo['orientation_error_deg']:.1f}°") - evidence.append(f"Anomaly ensemble score = {an['score']:.3f} " - f"(threshold {an['decision_threshold']:.2f}, strategy {an['ensemble_strategy']})") - if an.get("detector_scores"): - top = sorted(an["normalized_scores"].items(), key=lambda kv: -kv[1])[:3] - evidence.append("Highest detector votes: " + ", ".join(f"{k}={v:.2f}" for k, v in top)) - if gr.get("consensus") is not None: - evidence.append(f"Grader consensus = {gr['consensus']:.2%} across {gr['grader_count']} graders") - if rec["tracking"]["id_switch"]: - evidence.append("Tracking evidence shows an identity switch on this object's track") - if rec["tracking"]["fragmentation"]: - evidence.append("Tracking evidence shows track fragmentation") - primary = dec.get("primary_failure_reason") - if primary and primary in REASON_HINTS: - headline, cause = REASON_HINTS[primary] - likely_cause = f"{headline}: most consistent with {cause}." - confidence = 0.72 - for r in reasons[1:4]: - if r in REASON_HINTS: - factors.append(f"{r}: {REASON_HINTS[r][1]}") - investigation.extend([ - "Open the frame in HITL review and compare camera vs LiDAR evidence", - "Check neighboring frames of the same track for consistent behavior", - "Verify the reference ground-truth type and coverage before trusting the comparison", - ]) - - ev = ctx.get("rare_event") - if ev: - classification = f"Rare Event: {ev['scenario_type'].replace('_', ' ').title()}" - evidence.append(f"Scenario {ev['scenario_type']} severity={ev['severity']} " - f"rarity={ev['rarity_score']:.2f} anomaly={ev['anomaly_score']:.2f}") - for sensor, detail in ev.get("sensor_evidence", {}).items(): - evidence.append(f"{sensor}: {detail}") - likely_cause = "Genuine rare scenario rather than a labeling defect; value lies in adding it to training." - investigation.append("Verify the event, then include its frames in the next training dataset") - confidence = 0.68 - - reg = ctx.get("regression") - if reg and ctype == "regression": - classification = "Model Performance Regression" - for d in reg.get("deltas", [])[:6]: - if d["regressed"]: - evidence.append(f"{d['metric']}: {d['baseline']:.3f} → {d['current']:.3f} " - f"(Δ {d['delta']:+.3f}, tolerance {d['tolerance']})") - if reg.get("affected_classes"): - factors.append("Affected classes: " + ", ".join(reg["affected_classes"])) - likely_cause = ("Training-data distribution shift or label-quality drop between model versions; " - "the regressed metrics cluster in " + ", ".join(reg.get("kinds", ["performance"])) + ".") - investigation.extend([ - "Diff the training dataset lineage between the two model versions", - "Re-run the benchmark on the affected classes only", - "Inspect flagged labels from the newer model for systematic geometry errors", - ]) - confidence = 0.7 - - if not evidence: - evidence.append("No structured evidence was attached to this request.") - investigation.append("Select a specific annotation, rare event, or regression entry and retry") - - hypothesis = (f"HYPOTHESIS (not a conclusion): {likely_cause} " - "This is generated from deterministic evidence rules and requires human confirmation.") - - lines = [f"# {classification}", "", "## Observed evidence"] - lines += [f"- {e}" for e in evidence] - if factors: - lines += ["", "## Contributing factors"] + [f"- {f}" for f in factors] - lines += ["", "## Likely cause", likely_cause, "", "## Hypothesis", hypothesis, - "", "## Recommended investigation"] + [f"- {i}" for i in investigation] - lines += ["", f"_Advisory only — confidence {confidence:.0%}. The Copilot never changes metrics or overrides gates._"] - - return { - "analysis": "\n".join(lines), - "structured": { - "failure_classification": classification, - "observed_evidence": evidence, - "likely_cause": likely_cause, - "contributing_factors": factors, - "hypothesis": hypothesis, - "recommended_investigation": investigation, - "confidence": confidence, - }, - } - - -def explain(store: EvalStore, req: Dict) -> Dict: - """Try Ollama first; fall back to the deterministic offline analysis.""" - ctx = _gather_context(store, req) - offline = _offline_analysis(ctx) - - prompt = ( - "You are an advisory MITL Copilot for an autonomous-driving label evaluation platform. " - "You never change metrics or approve/reject labels; you explain evidence.\n\n" - f"CONTEXT TYPE: {ctx.get('context_type')}\n" - f"EVIDENCE (JSON):\n{json.dumps({k: v for k, v in ctx.items() if k != 'context_type'}, indent=2)[:6000]}\n\n" - "Produce: failure classification, observed evidence, likely cause, contributing factors, " - "a clearly-labeled hypothesis, recommended investigation steps, and a confidence estimate. " - "Be concise and concrete." - ) - for ep in OLLAMA_ENDPOINTS: - try: - res = httpx.post(ep["url"], json={ - "model": ep["model"], - "messages": [{"role": "user", "content": prompt}], - "stream": False, - }, timeout=20.0) - if res.status_code == 200: - text = res.json().get("message", {}).get("content", "") - if text: - return { - "status": "ok", - "provider": ep["url"], - "analysis": text, - "structured": offline["structured"], # deterministic evidence summary retained - } - except Exception: - continue - - return { - "status": "ok", - "provider": "offline_deterministic", - "analysis": offline["analysis"], - "structured": offline["structured"], - } diff --git a/hf-space/sensorflow/evaluation/detectors.py b/hf-space/sensorflow/evaluation/detectors.py deleted file mode 100644 index bf57d9fe46942c95e97cc2d89f8a26d7b27be8ed..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/detectors.py +++ /dev/null @@ -1,414 +0,0 @@ -"""ML anomaly detection engine: multiple detector families + score ensemble. - -Families (spec §13): -- density: KNN distance, Local Outlier Factor -- isolation: Isolation Forest, One-Class SVM, DBSCAN noise/centroid distance -- deep: Autoencoder reconstruction error (lightweight numpy MLP) -- generative: VAE (numpy, PCA-latent Gaussian likelihood), GAN (numpy - Mahalanobis "discriminator" surrogate) -- metric: few-shot prototype embeddings (support/query distance) - -The deep/generative detectors are intentionally lightweight numpy -implementations (no torch in this environment): clearly marked, deterministic, -and still produce meaningful scores on the synthetic features. - -All detectors implement Detector.fit(X) / .score(X) where HIGHER = more -anomalous. Ensemble normalizes scores before combining and records -detector_scores, normalized_scores, ensemble_strategy, ensemble_score and the -decision_threshold on every decision (spec §14). -""" - -from __future__ import annotations - -from typing import Dict, List, Optional, Tuple - -import numpy as np -from sklearn.cluster import DBSCAN -from sklearn.ensemble import IsolationForest -from sklearn.linear_model import LogisticRegression -from sklearn.neighbors import LocalOutlierFactor, NearestNeighbors -from sklearn.preprocessing import StandardScaler -from sklearn.svm import OneClassSVM - - -class Detector: - name = "detector" - family = "generic" - - def fit(self, X: np.ndarray) -> "Detector": - raise NotImplementedError - - def score(self, X: np.ndarray) -> np.ndarray: - """Higher = more anomalous.""" - raise NotImplementedError - - -class KNNDetector(Detector): - name = "knn" - family = "density" - - def __init__(self, k: int = 10): - self.k = k - self.nn: Optional[NearestNeighbors] = None - - def fit(self, X: np.ndarray) -> "KNNDetector": - self.nn = NearestNeighbors(n_neighbors=min(self.k + 1, len(X))) - self.nn.fit(X) - return self - - def score(self, X: np.ndarray) -> np.ndarray: - dists, _ = self.nn.kneighbors(X) - return dists[:, 1:].mean(axis=1) # skip self-distance - - -class LOFDetector(Detector): - name = "lof" - family = "density" - - def __init__(self, n_neighbors: int = 20): - self.n_neighbors = n_neighbors - self._scores: Optional[np.ndarray] = None - self._X: Optional[np.ndarray] = None - - def fit(self, X: np.ndarray) -> "LOFDetector": - lof = LocalOutlierFactor(n_neighbors=min(self.n_neighbors, max(2, len(X) - 1))) - lof.fit(X) - self._scores = -lof.negative_outlier_factor_ - self._X = X - return self - - def score(self, X: np.ndarray) -> np.ndarray: - if self._X is not None and X.shape == self._X.shape and np.allclose(X, self._X): - return self._scores - # Novel points (e.g. a re-scored corrected label): approximate LOF by - # the ratio of the query's kNN distance to the training population's. - nn = NearestNeighbors(n_neighbors=min(self.n_neighbors, len(self._X))) - nn.fit(self._X) - d_query, _ = nn.kneighbors(X) - d_train, _ = nn.kneighbors(self._X) - ref = max(float(d_train[:, 1:].mean()), 1e-9) - return d_query.mean(axis=1) / ref - - -class IsolationForestDetector(Detector): - name = "isolation_forest" - family = "isolation" - - def __init__(self, n_estimators: int = 100, seed: int = 7): - self.model = IsolationForest(n_estimators=n_estimators, random_state=seed) - - def fit(self, X: np.ndarray) -> "IsolationForestDetector": - self.model.fit(X) - return self - - def score(self, X: np.ndarray) -> np.ndarray: - return -self.model.score_samples(X) - - -class OCSVMDetector(Detector): - name = "ocsvm" - family = "isolation" - - def __init__(self, nu: float = 0.05): - self.model = OneClassSVM(nu=nu, kernel="rbf", gamma="scale") - - def fit(self, X: np.ndarray) -> "OCSVMDetector": - self.model.fit(X) - return self - - def score(self, X: np.ndarray) -> np.ndarray: - return -self.model.decision_function(X) - - -class DBSCANDetector(Detector): - name = "dbscan" - family = "isolation" - - def __init__(self, eps: float = 1.2, min_samples: int = 5): - self.eps = eps - self.min_samples = min_samples - self.centroids: Optional[np.ndarray] = None - - def fit(self, X: np.ndarray) -> "DBSCANDetector": - labels = DBSCAN(eps=self.eps, min_samples=self.min_samples).fit_predict(X) - cents = [X[labels == c].mean(axis=0) for c in sorted(set(labels)) if c != -1] - self.centroids = np.array(cents) if cents else X.mean(axis=0, keepdims=True) - return self - - def score(self, X: np.ndarray) -> np.ndarray: - d = np.linalg.norm(X[:, None, :] - self.centroids[None, :, :], axis=2) - return d.min(axis=1) - - -class AutoencoderDetector(Detector): - """Lightweight numpy MLP autoencoder (in-repo substitute for a torch AE). - - Single hidden bottleneck, tanh activation, plain SGD; deterministic seed. - Score = per-sample reconstruction MSE. - """ - - name = "autoencoder" - family = "deep" - - def __init__(self, latent_dim: int = 4, epochs: int = 60, lr: float = 0.05, seed: int = 7): - self.latent_dim = latent_dim - self.epochs = epochs - self.lr = lr - self.seed = seed - self.W1 = self.b1 = self.W2 = self.b2 = None - - def fit(self, X: np.ndarray) -> "AutoencoderDetector": - rng = np.random.default_rng(self.seed) - n, d = X.shape - k = min(self.latent_dim, d) - self.W1 = rng.normal(0, 0.1, (d, k)) - self.b1 = np.zeros(k) - self.W2 = rng.normal(0, 0.1, (k, d)) - self.b2 = np.zeros(d) - for _ in range(self.epochs): - idx = rng.permutation(n)[: min(n, 512)] - xb = X[idx] - z = np.tanh(xb @ self.W1 + self.b1) - xh = z @ self.W2 + self.b2 - err = xh - xb # (B, d) - gW2 = z.T @ err / len(xb) - gb2 = err.mean(axis=0) - dz = (err @ self.W2.T) * (1 - z ** 2) - gW1 = xb.T @ dz / len(xb) - gb1 = dz.mean(axis=0) - self.W2 -= self.lr * gW2 - self.b2 -= self.lr * gb2 - self.W1 -= self.lr * gW1 - self.b1 -= self.lr * gb1 - return self - - def score(self, X: np.ndarray) -> np.ndarray: - z = np.tanh(X @ self.W1 + self.b1) - xh = z @ self.W2 + self.b2 - return ((xh - X) ** 2).mean(axis=1) - - -class VAEDetector(Detector): - """Lightweight generative surrogate: Gaussian likelihood in a PCA latent - space (stands in for a torch VAE ELBO). Score = negative log-likelihood. - """ - - name = "vae" - family = "generative" - - def __init__(self, latent_dim: int = 5): - self.latent_dim = latent_dim - self.mean = self.components = self.latent_var = self.noise_var = None - - def fit(self, X: np.ndarray) -> "VAEDetector": - self.mean = X.mean(axis=0) - Xc = X - self.mean - _, s, vt = np.linalg.svd(Xc, full_matrices=False) - k = min(self.latent_dim, vt.shape[0]) - self.components = vt[:k] - z = Xc @ self.components.T - self.latent_var = z.var(axis=0) + 1e-6 - recon = z @ self.components - self.noise_var = max(float(((Xc - recon) ** 2).mean()), 1e-6) - return self - - def score(self, X: np.ndarray) -> np.ndarray: - Xc = X - self.mean - z = Xc @ self.components.T - recon = z @ self.components - nll_latent = 0.5 * ((z ** 2) / self.latent_var).sum(axis=1) - nll_recon = 0.5 * ((Xc - recon) ** 2).sum(axis=1) / self.noise_var - return nll_latent + nll_recon - - -class GANDetector(Detector): - """Lightweight generative-adversarial surrogate: discriminator score - approximated by Mahalanobis distance under a shrunk covariance of the - "real" data distribution (stands in for a torch GAN discriminator). - """ - - name = "gan" - family = "generative" - - def __init__(self, shrinkage: float = 0.1): - self.shrinkage = shrinkage - self.mean = self.prec = None - - def fit(self, X: np.ndarray) -> "GANDetector": - self.mean = X.mean(axis=0) - cov = np.cov(X, rowvar=False) - cov = (1 - self.shrinkage) * cov + self.shrinkage * np.eye(X.shape[1]) * np.trace(cov) / X.shape[1] - self.prec = np.linalg.pinv(cov) - return self - - def score(self, X: np.ndarray) -> np.ndarray: - d = X - self.mean - return np.sqrt(np.maximum((d @ self.prec * d).sum(axis=1), 0)) - - -class FewShotDetector(Detector): - """Metric-learning detector: class prototypes from a small support set of - presumed-normal samples; query score = distance to nearest prototype. - """ - - name = "few_shot" - family = "metric" - - def __init__(self, support_per_class: int = 20, n_prototypes: int = 8, seed: int = 7): - self.support_per_class = support_per_class - self.n_prototypes = n_prototypes - self.seed = seed - self.prototypes: Optional[np.ndarray] = None - - def fit(self, X: np.ndarray) -> "FewShotDetector": - rng = np.random.default_rng(self.seed) - # Support set: samples nearest to the data median (most "typical"). - med = np.median(X, axis=0) - d = np.linalg.norm(X - med, axis=1) - order = np.argsort(d) - support = X[order[: max(self.n_prototypes * self.support_per_class, 10)]] - # Simple k-means style prototypes over the support set. - k = min(self.n_prototypes, len(support)) - protos = support[rng.choice(len(support), k, replace=False)] - for _ in range(10): - assign = np.argmin(np.linalg.norm(support[:, None] - protos[None], axis=2), axis=1) - for j in range(k): - pts = support[assign == j] - if len(pts): - protos[j] = pts.mean(axis=0) - self.prototypes = protos - return self - - def score(self, X: np.ndarray) -> np.ndarray: - d = np.linalg.norm(X[:, None, :] - self.prototypes[None, :, :], axis=2) - return d.min(axis=1) - - -# ------------------------------------------------------------------ ensemble - - -def normalize_scores(scores: np.ndarray) -> np.ndarray: - """Rank-based normalization to [0, 1]; robust to scale differences.""" - order = scores.argsort().argsort().astype(np.float64) - denom = max(len(scores) - 1, 1) - return order / denom - - -DEFAULT_WEIGHTS = { - "knn": 1.0, - "lof": 1.0, - "isolation_forest": 1.2, - "ocsvm": 0.8, - "dbscan": 0.8, - "autoencoder": 1.0, - "vae": 0.9, - "gan": 0.7, - "few_shot": 0.9, -} - - -class AnomalyEnsemble: - """Runs enabled detectors on standardized features and combines them.""" - - def __init__(self, config: Optional[Dict] = None, seed: int = 7): - self.config = config or {} - self.seed = seed - self.detectors = self._build() - self.scaler = StandardScaler() - - def _cfg(self, *path, default=None): - node = self.config - for p in path: - if not isinstance(node, dict) or p not in node: - return default - node = node[p] - return node - - def _build(self) -> List[Detector]: - dets: List[Detector] = [] - if self._cfg("detectors", "knn", "enabled", default=True): - dets.append(KNNDetector(k=int(self._cfg("detectors", "knn", "k", default=10)))) - if self._cfg("detectors", "lof", "enabled", default=True): - dets.append(LOFDetector(n_neighbors=int(self._cfg("detectors", "lof", "n_neighbors", default=20)))) - if self._cfg("detectors", "isolation_forest", "enabled", default=True): - dets.append(IsolationForestDetector( - n_estimators=int(self._cfg("detectors", "isolation_forest", "n_estimators", default=100)), - seed=self.seed)) - if self._cfg("detectors", "ocsvm", "enabled", default=True): - dets.append(OCSVMDetector(nu=float(self._cfg("detectors", "ocsvm", "nu", default=0.05)))) - if self._cfg("detectors", "dbscan", "enabled", default=True): - dets.append(DBSCANDetector( - eps=float(self._cfg("detectors", "dbscan", "eps", default=1.2)), - min_samples=int(self._cfg("detectors", "dbscan", "min_samples", default=5)))) - if self._cfg("deep", "autoencoder", "enabled", default=True): - dets.append(AutoencoderDetector( - latent_dim=int(self._cfg("deep", "autoencoder", "latent_dim", default=4)), - epochs=int(self._cfg("deep", "autoencoder", "epochs", default=60)), - seed=self.seed)) - if self._cfg("deep", "vae", "enabled", default=True): - dets.append(VAEDetector(latent_dim=int(self._cfg("deep", "vae", "latent_dim", default=5)))) - if self._cfg("deep", "gan", "enabled", default=False): - dets.append(GANDetector()) - if self._cfg("advanced", "few_shot", "enabled", default=True): - dets.append(FewShotDetector( - support_per_class=int(self._cfg("advanced", "few_shot", "support_per_class", default=20)), - seed=self.seed)) - return dets - - @property - def strategy(self) -> str: - return str(self._cfg("advanced", "ensemble_strategy", default="weighted_average")) - - @property - def threshold(self) -> float: - return float(self._cfg("advanced", "decision_threshold", default=0.9)) - - def run( - self, - X: np.ndarray, - supervision: Optional[np.ndarray] = None, - ) -> Tuple[np.ndarray, Dict[str, np.ndarray], Dict[str, np.ndarray]]: - """Returns (ensemble_scores, raw_scores_by_detector, normalized_by_detector). - - supervision (optional 0/1 labels) is used ONLY by the meta_classifier - strategy, trained on a held-out half to avoid trivially memorizing. - """ - Xs = self.scaler.fit_transform(X) - raw: Dict[str, np.ndarray] = {} - norm: Dict[str, np.ndarray] = {} - for det in self.detectors: - try: - det.fit(Xs) - s = det.score(Xs) - except Exception: - s = np.zeros(len(Xs)) - raw[det.name] = s - norm[det.name] = normalize_scores(s) - - if not norm: - return np.zeros(len(X)), raw, norm - - M = np.stack([norm[k] for k in norm], axis=1) - strategy = self.strategy - if strategy == "majority_vote": - votes = (M >= self.threshold).astype(float) - ensemble = votes.mean(axis=1) - elif strategy == "meta_classifier" and supervision is not None and supervision.sum() >= 4: - rng = np.random.default_rng(self.seed) - idx = rng.permutation(len(M)) - half = len(M) // 2 - tr, te = idx[:half], idx[half:] - clf = LogisticRegression(max_iter=500) - ensemble = np.zeros(len(M)) - try: - clf.fit(M[tr], supervision[tr]) - ensemble[te] = clf.predict_proba(M[te])[:, 1] - clf2 = LogisticRegression(max_iter=500).fit(M[te], supervision[te]) - ensemble[tr] = clf2.predict_proba(M[tr])[:, 1] - except Exception: - ensemble = M.mean(axis=1) - else: - weights = np.array([DEFAULT_WEIGHTS.get(k, 1.0) for k in norm]) - ensemble = (M * weights).sum(axis=1) / weights.sum() - - return ensemble, raw, norm diff --git a/hf-space/sensorflow/evaluation/graders.py b/hf-space/sensorflow/evaluation/graders.py deleted file mode 100644 index ae9eb8c2d18c0cbd6e667b2bbd74b26501bdf13d..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/graders.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Grader disagreement subsystem. - -Grader sources (spec §11): auto label, vendor GT, historical model, VLM label, -human reviewer (when present), independent detector. Computes classification, -spatial, temporal and statistical agreement, and a canonical -grader_consensus_score while preserving the components. - -Statistical agreement — used correctly, not interchangeably: -- Cohen's Kappa: exactly 2 graders, categorical labels. -- Fleiss' Kappa: >2 graders, every grader rates every (fixed) item. -- Krippendorff's Alpha: any number of graders, tolerates missing ratings. -""" - -from __future__ import annotations - -import math -from collections import Counter -from typing import Dict, List, Optional, Sequence, Tuple - -import numpy as np - -from sensorflow.evaluation.records import ( - Annotation, - EvalStore, - Frame, - GraderComparison, -) -from sensorflow.metrics.perception_3d import bev_iou - -GRADER_SOURCES = ["auto_label", "vendor_gt", "historical_model", "vlm_label", "independent_detector"] - -CLASS_LIST = ["pedestrian", "cyclist", "vehicle", "motorcycle", "truck"] - - -# ------------------------------------------------------------------ statistics - - -def cohens_kappa(a: Sequence[str], b: Sequence[str]) -> float: - """Cohen's kappa for exactly two graders over the same items.""" - assert len(a) == len(b) and len(a) > 0 - cats = sorted(set(a) | set(b)) - n = len(a) - po = sum(1 for x, y in zip(a, b) if x == y) / n - pa = Counter(a) - pb = Counter(b) - pe = sum((pa[c] / n) * (pb[c] / n) for c in cats) - if math.isclose(pe, 1.0): - return 1.0 - return (po - pe) / (1 - pe) - - -def fleiss_kappa(ratings: List[Dict[str, int]]) -> float: - """Fleiss' kappa: ratings[i] = {category: count} with a fixed rater count - per item (every grader rates every item).""" - if not ratings: - return 0.0 - n_raters = sum(ratings[0].values()) - cats = sorted({c for r in ratings for c in r}) - N = len(ratings) - p_j = {c: sum(r.get(c, 0) for r in ratings) / (N * n_raters) for c in cats} - P_i = [] - for r in ratings: - s = sum(cnt * (cnt - 1) for cnt in r.values()) - P_i.append(s / (n_raters * (n_raters - 1)) if n_raters > 1 else 1.0) - P_bar = float(np.mean(P_i)) - P_e = sum(p ** 2 for p in p_j.values()) - if math.isclose(P_e, 1.0): - return 1.0 - return (P_bar - P_e) / (1 - P_e) - - -def krippendorff_alpha(data: List[List[Optional[str]]]) -> float: - """Krippendorff's alpha (nominal) tolerating missing ratings. - - data[rater][item] = category or None. - """ - # Collect pairable values per item (unit). - units: List[List[str]] = [] - n_items = max((len(r) for r in data), default=0) - for i in range(n_items): - vals = [r[i] for r in data if i < len(r) and r[i] is not None] - if len(vals) >= 2: - units.append(vals) - if not units: - return 0.0 - # Observed disagreement. - Do_num, Do_den = 0.0, 0.0 - all_vals: List[str] = [] - for vals in units: - m = len(vals) - all_vals.extend(vals) - cnt = Counter(vals) - pairs_disagree = 0 - for c1 in cnt: - for c2 in cnt: - if c1 != c2: - pairs_disagree += cnt[c1] * cnt[c2] - Do_num += pairs_disagree / (m - 1) - Do_den += m - Do = Do_num / Do_den if Do_den else 0.0 - # Expected disagreement. - total = len(all_vals) - cnt_all = Counter(all_vals) - De_num = 0.0 - for c1 in cnt_all: - for c2 in cnt_all: - if c1 != c2: - De_num += cnt_all[c1] * cnt_all[c2] - De = De_num / (total * (total - 1)) if total > 1 else 0.0 - if math.isclose(De, 0.0): - return 1.0 - return 1 - Do / De - - -# ------------------------------------------------------------------ simulated graders - -def _simulate_grader_votes(ann: Annotation, gt_class: Optional[str], rng: np.random.Generator) -> Dict[str, str]: - """Deterministic per-annotation grader simulation. - - Each grader source produces a class vote. Injected GRADER_DISAGREEMENT - forces the VLM + independent detector to disagree. - """ - base = gt_class or ann.class_name - votes: Dict[str, str] = {"auto_label": ann.class_name} - votes["vendor_gt"] = gt_class if gt_class else ann.class_name - - def maybe_flip(cls: str, p: float) -> str: - if rng.random() < p: - alts = [c for c in CLASS_LIST if c != cls] - return alts[int(rng.integers(0, len(alts)))] - return cls - - disagree = "GRADER_DISAGREEMENT" in ann.injected_errors - votes["historical_model"] = maybe_flip(base, 0.02) - votes["vlm_label"] = maybe_flip(base, 1.0 if disagree else 0.04) - votes["independent_detector"] = maybe_flip(base, 1.0 if disagree else 0.03) - return votes - - -def _spatial_agreement(ann: Annotation, gt_box: Optional[List[float]], rng: np.random.Generator) -> Tuple[float, Dict[str, float]]: - """Spatial agreement between graders' boxes (auto vs reference vs an - independent perturbation): IoU, center distance, dims, orientation.""" - if not ann.bbox_3d: - return 0.0, {} - ref = gt_box if gt_box else ann.bbox_3d - iou = bev_iou(ann.bbox_3d, ref) - center_dist = math.hypot(ann.bbox_3d[0] - ref[0], ann.bbox_3d[1] - ref[1]) - dim_diff = float(np.mean([abs(ann.bbox_3d[i] - ref[i]) / max(ref[i], 0.1) for i in (3, 4, 5)])) - yaw_diff = abs(ann.bbox_3d[6] - ref[6]) % (2 * math.pi) - yaw_diff = min(yaw_diff, 2 * math.pi - yaw_diff) - detail = { - "iou": round(iou, 4), - "center_distance": round(center_dist, 3), - "dimension_diff": round(dim_diff, 4), - "orientation_diff_deg": round(math.degrees(yaw_diff), 2), - } - score = float(np.clip( - 0.5 * iou + 0.25 * max(0, 1 - center_dist / 2.0) + 0.15 * max(0, 1 - dim_diff) + 0.10 * max(0, 1 - yaw_diff / math.pi), - 0, 1)) - return score, detail - - -def grade_annotation( - store: EvalStore, - ann: Annotation, - frame: Frame, - temporal_quality: Optional[float] = None, - seed: int = 7, -) -> GraderComparison: - rng = np.random.default_rng(abs(hash((ann.annotation_id, seed))) % (2 ** 32)) - gt = next((g for g in frame.gt_boxes if g.gt_id == ann.matched_gt_id), None) - - votes = _simulate_grader_votes(ann, gt.class_name if gt else None, rng) - classes = list(votes.values()) - # Reliability-weighted class agreement: authoritative sources count more. - weights = {"auto_label": 1.0, "vendor_gt": 2.0, "historical_model": 0.75, - "vlm_label": 0.5, "independent_detector": 0.75} - weight_by_class: Dict[str, float] = {} - for source, cls in votes.items(): - weight_by_class[cls] = weight_by_class.get(cls, 0.0) + weights.get(source, 1.0) - total_weight = sum(weights.get(s, 1.0) for s in votes) - class_agreement = max(weight_by_class.values()) / total_weight - - spatial, spatial_detail = _spatial_agreement(ann, gt.bbox_3d if gt else None, rng) - temporal = temporal_quality if temporal_quality is not None else (1.0 if ann.track_id else 0.5) - - # Per-item agreement indicators; proper kappa/alpha statistics are computed - # at dataset level in dataset_grader_statistics (kappa is undefined on a - # single item). - kappa_stats: Dict[str, float] = { - "auto_vs_vendor_agree": 1.0 if votes["auto_label"] == votes["vendor_gt"] else 0.0, - "vote_agreement": round(class_agreement, 4), - } - - disagreements: List[str] = [] - if class_agreement < 1.0: - disagreements.append("classification") - if spatial < 0.6: - disagreements.append("spatial") - if temporal < 0.6: - disagreements.append("temporal") - - consensus = float(np.clip(0.5 * class_agreement + 0.35 * spatial + 0.15 * temporal, 0, 1)) - - cmp = GraderComparison( - annotation_id=ann.annotation_id, - grader_count=len(votes), - graders=list(votes.keys()), - class_votes=votes, - class_agreement=round(class_agreement, 4), - spatial_agreement=round(spatial, 4), - temporal_agreement=round(float(temporal), 4), - consensus=round(consensus, 4), - disagreement_types=disagreements, - kappa_stats={**kappa_stats, **{f"spatial_{k}": v for k, v in spatial_detail.items()}}, - ) - store.put("grader_comparisons", cmp) - return cmp - - -def dataset_grader_statistics(store: EvalStore, dataset_id: str) -> Dict[str, float]: - """Dataset-level statistical agreement across all graded items.""" - anns = store.where("annotations", dataset_id=dataset_id) - comparisons = [store.get("grader_comparisons", a.annotation_id) for a in anns] - comparisons = [c for c in comparisons if c is not None] - if not comparisons: - return {} - - # Two-grader Cohen's kappa: auto label vs vendor GT over all items. - auto = [c.class_votes.get("auto_label", "") for c in comparisons] - vendor = [c.class_votes.get("vendor_gt", "") for c in comparisons] - cohen = cohens_kappa(auto, vendor) - - # Fleiss' kappa: all graders rate all items (fixed panel). - ratings = [dict(Counter(c.class_votes.values())) for c in comparisons] - fleiss = fleiss_kappa(ratings) - - # Krippendorff's alpha: tolerate a missing vendor rating (simulated missing - # data on unmatched labels). - raters: List[List[Optional[str]]] = [ - [c.class_votes.get(source) for c in comparisons] for source in GRADER_SOURCES - ] - # Mark vendor rating missing where the label had no matched GT. - ann_by_id = {a.annotation_id: a for a in anns} - for j, c in enumerate(comparisons): - a = ann_by_id.get(c.annotation_id) - if a is not None and a.matched_gt_id is None: - raters[GRADER_SOURCES.index("vendor_gt")][j] = None - kripp = krippendorff_alpha(raters) - - return { - "cohen_kappa": round(cohen, 4), - "fleiss_kappa": round(fleiss, 4), - "krippendorff_alpha": round(kripp, 4), - "mean_consensus": round(float(np.mean([c.consensus for c in comparisons if c.consensus is not None])), 4), - } diff --git a/hf-space/sensorflow/evaluation/pipeline.py b/hf-space/sensorflow/evaluation/pipeline.py deleted file mode 100644 index f96ef877f6478e343d543aa2bd2a0956fb1fea5b..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/pipeline.py +++ /dev/null @@ -1,836 +0,0 @@ -"""Pipeline orchestrator: logical processing services over a job queue. - -Services (spec §5): label-generator, anomaly-detector, regression-tracker, -grader, quality-validator, rare-event-detector, scenario-miner, triage-engine, -review-service, metrics-service — modular workers inside one app, surfaced to -the UI as logical services with live status. - -Flow: Ingestion -> Label Generation -> Queue -> Evaluation workers -> -Validation workers -> Grading workers -> Triage -> (Auto-grade | HITL). -""" - -from __future__ import annotations - -import math -import threading -import time -from typing import Dict, List, Optional - -import numpy as np - -from sensorflow.evaluation import synthetic -from sensorflow.evaluation.detectors import AnomalyEnsemble -from sensorflow.evaluation.graders import dataset_grader_statistics, grade_annotation -from sensorflow.evaluation.process_units import ProcessMeter -from sensorflow.evaluation.queue import EventQueue, make_queue -from sensorflow.evaluation.rare_events import detect_rare_events -from sensorflow.evaluation.records import ( - Alert, - Annotation, - AnomalyDetection, - DatasetLineage, - EvalStore, - Model, - ModelEvaluation, - RelabelingAction, - ReviewResolution, - ReviewTask, - HumanReview, - TrackingEvidence, - TrainingDataset, - TrainingJob, - get_store, - new_id, -) -from sensorflow.evaluation.regression import compare_runs, find_affected_classes -from sensorflow.evaluation.triage import QualityPolicy, load_policy, triage_annotation -from sensorflow.evaluation.validation import validate_annotation -from sensorflow.evaluation import reporting - -SERVICES = [ - "label-generator", - "anomaly-detector", - "regression-tracker", - "grader", - "quality-validator", - "rare-event-detector", - "scenario-miner", - "triage-engine", - "review-service", - "metrics-service", -] - -DEFAULT_ANOMALY_CONFIG: Dict = { - "imbalance": {"method": "class_weights", "minority_boost": 2.0}, - "detectors": { - "knn": {"enabled": True, "k": 10}, - "lof": {"enabled": True, "n_neighbors": 20}, - "isolation_forest": {"enabled": True, "n_estimators": 100}, - "ocsvm": {"enabled": True, "nu": 0.05}, - "dbscan": {"enabled": True, "eps": 1.2, "min_samples": 5}, - }, - "deep": { - "autoencoder": {"enabled": True, "latent_dim": 4, "epochs": 60}, - "vae": {"enabled": True, "latent_dim": 5}, - "gan": {"enabled": False}, - "reconstruction_threshold": 0.9, - }, - "advanced": { - "few_shot": {"enabled": True, "support_per_class": 20}, - "ensemble_strategy": "weighted_average", - "decision_threshold": 0.9, - }, -} - -BATCH_SIZE = 64 -BATCH_PAUSE_S = 0.015 # keeps the live queue/SSE view meaningful without slowing tests much - - -class LabelEvalPipeline: - """Singleton orchestrator holding queue, services and live run state.""" - - def __init__(self, store: Optional[EvalStore] = None, queue: Optional[EventQueue] = None): - self.store = store or get_store() - self.queue = queue or make_queue("memory") - self.lock = threading.RLock() - self.services: Dict[str, Dict] = { - s: {"state": "IDLE", "processed": 0, "total": 0, "detail": "", "process_units": 0} - for s in SERVICES - } - self.running = False - self.stage = "idle" - self.last_run_id: Optional[str] = None - self.active_dataset: Optional[str] = None - self.anomaly_config: Dict = dict(DEFAULT_ANOMALY_CONFIG) - self.regression_alert = False - self._thread: Optional[threading.Thread] = None - self._train_threads: Dict[str, threading.Thread] = {} - self._score_context: Optional[Dict] = None # for re-scoring corrected labels - # restore persisted light state - meta = self.store.meta - self.active_dataset = meta.get("active_dataset") - self.last_run_id = meta.get("last_run_id") - self.anomaly_config = meta.get("anomaly_config", dict(DEFAULT_ANOMALY_CONFIG)) - self.regression_alert = bool(meta.get("regression_alert", False)) - - # ------------------------------------------------------------ service status - - def _svc(self, name: str, state: Optional[str] = None, processed: Optional[int] = None, - total: Optional[int] = None, detail: Optional[str] = None, add_units: int = 0) -> None: - with self.lock: - svc = self.services[name] - if state is not None: - svc["state"] = state - if processed is not None: - svc["processed"] = processed - if total is not None: - svc["total"] = total - if detail is not None: - svc["detail"] = detail - svc["process_units"] += add_units - - def service_status(self) -> List[Dict]: - with self.lock: - return [{"service": s, **self.services[s]} for s in SERVICES] - - # ------------------------------------------------------------ run orchestration - - def run(self, dataset_id: str, background: bool = True, policy: Optional[QualityPolicy] = None) -> str: - with self.lock: - if self.running: - raise RuntimeError("A pipeline run is already in progress") - self.running = True - run_id = new_id("run") - self.last_run_id = run_id - self.active_dataset = dataset_id - if background: - self._thread = threading.Thread(target=self._run_safe, args=(dataset_id, run_id, policy), daemon=True) - self._thread.start() - else: - self._run_safe(dataset_id, run_id, policy) - return run_id - - def _run_safe(self, dataset_id: str, run_id: str, policy: Optional[QualityPolicy]) -> None: - try: - self._execute(dataset_id, run_id, policy or load_policy()) - except Exception as exc: # surface failures as service state, not silent death - self.stage = "failed" - for s in SERVICES: - if self.services[s]["state"] == "RUNNING": - self._svc(s, state="FAILED", detail=str(exc)[:200]) - self.store.audit("pipeline_failed", "Run", run_id, str(exc)[:400]) - finally: - with self.lock: - self.running = False - self.store.meta.update({ - "active_dataset": self.active_dataset, - "last_run_id": self.last_run_id, - "anomaly_config": self.anomaly_config, - "regression_alert": self.regression_alert, - }) - self.store.save() - - def _execute(self, dataset_id: str, run_id: str, policy: QualityPolicy) -> None: - store = self.store - meter = ProcessMeter(store, run_id) - dataset = store.get("datasets", dataset_id) - if dataset is None: - raise ValueError(f"Unknown dataset {dataset_id}") - self.queue.reset() - - # ---- 1. ingestion - self.stage = "ingestion" - frames = sorted(store.where("frames", dataset_id=dataset_id), key=lambda f: f.index) - units = meter.record("ingestion", len(frames)) - self._svc("label-generator", state="RUNNING", processed=0, total=len(frames), - detail="ingesting sensor frames", add_units=units) - - # ---- 2. label generation (labels are hypotheses; skip if already labeled) - self.stage = "label_generation" - annotations = store.where("annotations", dataset_id=dataset_id) - model_version = self._active_model_version() - if not annotations: - annotations = synthetic.generate_labels(store, dataset, model_version=model_version) - else: - model_version = annotations[0].model_version - # Fresh evaluation: reset transient statuses of non-human-reviewed labels. - for a in annotations: - if a.source == "auto": - a.status = "PENDING" - store.put("annotations", a) - units = meter.record("label_generation", len(annotations)) - self._svc("label-generator", state="HEALTHY", processed=len(frames), total=len(frames), - detail=f"{len(annotations)} labels by {model_version}", add_units=units) - - # ---- 3. enqueue for evaluation - self.stage = "queueing" - for ann in annotations: - self.queue.publish("evaluation", {"annotation_id": ann.annotation_id}) - - frames_by_id = {f.frame_id: f for f in frames} - ann_by_id = {a.annotation_id: a for a in annotations} - - # ---- 4. anomaly detection (fit once on the full population) - self.stage = "anomaly_detection" - self._svc("anomaly-detector", state="RUNNING", processed=0, total=len(annotations), - detail="fitting detector ensemble") - X, ids = synthetic.annotation_features(store, dataset_id) - supervision = np.array([1.0 if ann_by_id[i].injected_errors else 0.0 for i in ids]) if len(ids) else None - ensemble = AnomalyEnsemble(self.anomaly_config, seed=dataset.seed) - scores, raw, norm = ensemble.run(X, supervision=supervision) - strategy = ensemble.strategy - threshold = 0.5 if strategy == "majority_vote" else ensemble.threshold - anomaly_by_id: Dict[str, AnomalyDetection] = {} - for j, aid in enumerate(ids): - det = AnomalyDetection( - annotation_id=aid, - score=round(float(scores[j]), 4), - is_anomaly=bool(scores[j] >= threshold), - detector_scores={k: round(float(v[j]), 4) for k, v in raw.items()}, - normalized_scores={k: round(float(v[j]), 4) for k, v in norm.items()}, - ensemble_strategy=strategy, - ensemble_score=round(float(scores[j]), 4), - decision_threshold=threshold, - ) - anomaly_by_id[aid] = det - store.put("anomalies", det) - self._score_context = { - "scaler": ensemble.scaler, - "detectors": ensemble.detectors, - "raw": raw, - "strategy": strategy, - "threshold": threshold, - } - units = meter.record("anomaly_detection", len(ids)) - self._svc("anomaly-detector", state="HEALTHY", processed=len(ids), total=len(ids), - detail=f"{sum(1 for d in anomaly_by_id.values() if d.is_anomaly)} anomalies " - f"({strategy})", add_units=units) - - # ---- 5. queue-driven evaluation -> validation -> grading workers - self.stage = "validation" - self._svc("quality-validator", state="RUNNING", processed=0, total=len(annotations)) - self._svc("grader", state="RUNNING", processed=0, total=len(annotations)) - points_cache: Dict[str, np.ndarray] = {} - tracking_by_id = self._tracking_evidence(annotations, frames_by_id) - processed_v = processed_g = 0 - - while True: - batch = self.queue.consume("evaluation", BATCH_SIZE) - if not batch: - break - for msg in batch: - self.queue.publish("validation", msg) - self.queue.ack("evaluation", len(batch)) - - vbatch = self.queue.consume("validation", BATCH_SIZE) - for msg in vbatch: - ann = ann_by_id[msg["annotation_id"]] - frame = frames_by_id[ann.frame_id] - if ann.frame_id not in points_cache: - points_cache[ann.frame_id] = synthetic.frame_points(store, frame) - validate_annotation(store, ann, frame, points=points_cache[ann.frame_id]) - processed_v += 1 - self.queue.publish("grading", msg) - self.queue.ack("validation", len(vbatch)) - self._svc("quality-validator", processed=processed_v) - - gbatch = self.queue.consume("grading", BATCH_SIZE) - for msg in gbatch: - ann = ann_by_id[msg["annotation_id"]] - frame = frames_by_id[ann.frame_id] - tev = tracking_by_id.get(ann.annotation_id) - grade_annotation(store, ann, frame, - temporal_quality=tev.track_quality if tev else None, - seed=dataset.seed) - processed_g += 1 - self.queue.publish("triage", msg) - self.queue.ack("grading", len(gbatch)) - self._svc("grader", processed=processed_g) - time.sleep(BATCH_PAUSE_S) - - units = meter.record("validation", processed_v) - self._svc("quality-validator", state="HEALTHY", detail="geometric validation complete", add_units=units) - units = meter.record("grading", processed_g) - stats = dataset_grader_statistics(store, dataset_id) - self._svc("grader", state="HEALTHY", - detail=f"kappa(C)={stats.get('cohen_kappa', 0):.2f} fleiss={stats.get('fleiss_kappa', 0):.2f} " - f"alpha={stats.get('krippendorff_alpha', 0):.2f}", add_units=units) - self.store.meta.setdefault("grader_stats", {})[dataset_id] = stats - - # ---- 6. rare events + scenario mining - self.stage = "rare_events" - self._svc("rare-event-detector", state="RUNNING", detail="mining scenarios") - self._svc("scenario-miner", state="RUNNING") - frame_scores: Dict[str, List[float]] = {} - for aid, det in anomaly_by_id.items(): - frame_scores.setdefault(ann_by_id[aid].frame_id, []).append(det.score) - mean_frame_scores = {fid: float(np.mean(v)) for fid, v in frame_scores.items()} - events = detect_rare_events(store, dataset_id, mean_frame_scores) - units = meter.record("rare_event_detection", len(frames)) - self._svc("rare-event-detector", state="HEALTHY", processed=len(events), total=len(events), - detail=f"{len(events)} rare events", add_units=units) - n_scen = len(store.where("scenarios", dataset_id=dataset_id)) - self._svc("scenario-miner", state="HEALTHY", processed=n_scen, total=n_scen, - detail=f"{n_scen} scenario groups") - - # ---- 7. regression tracking (vs previous run baseline) - self.stage = "regression" - self._svc("regression-tracker", state="RUNNING") - current = reporting.quality_metrics(store, dataset_id) - cur_global = current["global"] - cur_per_class = {c["class_name"]: c for c in current["per_class"]} - baseline = self.store.meta.get("baseline_metrics") - affected: List[str] = [] - if baseline: - affected = find_affected_classes(cur_per_class, baseline.get("per_class", {})) - reg = compare_runs( - store, cur_global, model_version, dataset.version, run_id, - baseline_metrics=baseline.get("global") if baseline else None, - baseline_version=baseline.get("model_version") if baseline else None, - affected_classes=affected, - affected_scenarios=[s["scenario"] for s in current["per_scenario"] - if s.get("recall") is not None and s["recall"] < 0.5], - ) - self.regression_alert = reg.regression_detected - units = meter.record("regression_tracking", len(annotations)) - self._svc("regression-tracker", state="DEGRADED" if reg.regression_detected else "HEALTHY", - detail="REGRESSION DETECTED" if reg.regression_detected else - f"stable vs {reg.baseline_version or 'no baseline'}", add_units=units) - # Current run becomes the next baseline. - self.store.meta["baseline_metrics"] = { - "global": cur_global, "per_class": cur_per_class, "model_version": model_version, - } - - # ---- 8. triage - self.stage = "triage" - self._svc("triage-engine", state="RUNNING", processed=0, total=len(annotations)) - flagged: List[Annotation] = [] - processed_t = 0 - while True: - batch = self.queue.consume("triage", BATCH_SIZE) - if not batch: - break - for msg in batch: - ann = ann_by_id[msg["annotation_id"]] - decision = triage_annotation( - store, ann, - validation=store.get("validations", ann.annotation_id), - anomaly=store.get("anomalies", ann.annotation_id), - grading=store.get("grader_comparisons", ann.annotation_id), - tracking=tracking_by_id.get(ann.annotation_id), - policy=policy, - model_regressed=reg.regression_detected, - ) - if decision.status == "AUTO_GRADED": - ann.status = "VERIFIED" # auto-release into the verified pool - else: - ann.status = "FLAGGED" - flagged.append(ann) - store.put("annotations", ann) - processed_t += 1 - self.queue.ack("triage", len(batch)) - self._svc("triage-engine", processed=processed_t) - self._svc("triage-engine", state="HEALTHY", - detail=f"{processed_t - len(flagged)} auto-graded / {len(flagged)} flagged") - - # ---- 9. review tasks for flagged labels - self.stage = "review_dispatch" - self._svc("review-service", state="RUNNING") - batch_id = f"review-batch-{run_id[-6:]}" - open_tasks = {t.annotation_id for t in store.all("review_tasks") if t.status != "resolved"} - created = 0 - for ann in flagged: - if ann.annotation_id in open_tasks: - continue - decision = store.get("triage_decisions", ann.annotation_id) - task = ReviewTask( - task_id=new_id("task"), - annotation_id=ann.annotation_id, - frame_id=ann.frame_id, - dataset_id=dataset_id, - failure_reasons=decision.failure_reasons if decision else [], - primary_failure_reason=decision.primary_failure_reason if decision else None, - review_batch=batch_id, - ) - store.put("review_tasks", task) - created += 1 - n_open = len([t for t in store.all("review_tasks") if t.status != "resolved"]) - self._svc("review-service", state="HEALTHY", processed=n_open, total=n_open, - detail=f"{created} new tasks (batch {batch_id})") - - # ---- 10. metrics + alerts - self.stage = "metrics" - self._svc("metrics-service", state="RUNNING") - self._refresh_alerts(dataset_id) - dataset.status = "evaluated" - store.put("datasets", dataset) - store.audit("pipeline_completed", "Run", run_id, - f"dataset={dataset_id} model={model_version} labels={len(annotations)} flagged={len(flagged)}") - self._svc("metrics-service", state="HEALTHY", detail="aggregates refreshed") - self.stage = "complete" - - # ------------------------------------------------------------ tracking evidence - - def _tracking_evidence(self, annotations: List[Annotation], frames_by_id: Dict) -> Dict[str, TrackingEvidence]: - """Detect ID switches and fragmentation from label tracks vs GT instances.""" - store = self.store - # GT instance -> ordered (frame_index, annotation) - by_instance: Dict[str, List] = {} - track_instances: Dict[str, set] = {} - for a in annotations: - if a.matched_gt_id is None or a.track_id is None: - continue - frame = frames_by_id.get(a.frame_id) - gt = next((g for g in frame.gt_boxes if g.gt_id == a.matched_gt_id), None) if frame else None - if gt is None: - continue - by_instance.setdefault(gt.track_instance_id, []).append((frame.index, a)) - track_instances.setdefault(a.track_id, set()).add(gt.track_instance_id) - - evidence: Dict[str, TrackingEvidence] = {} - # A pred track spanning multiple GT instances => identity contamination. - contaminated_tracks = {t for t, insts in track_instances.items() if len(insts) > 1} - for inst, items in by_instance.items(): - items.sort(key=lambda p: p[0]) - track_ids = [a.track_id for _, a in items] - distinct = set(track_ids) - fragmented = len(distinct) > 1 - prev_tid = None - for idx, (_, a) in enumerate(items): - switched = prev_tid is not None and a.track_id != prev_tid - prev_tid = a.track_id - contaminated = a.track_id in contaminated_tracks - quality = 1.0 - if fragmented: - quality -= 0.3 - if switched or contaminated: - quality -= 0.4 - ev = TrackingEvidence( - annotation_id=a.annotation_id, - id_switch=bool(switched or contaminated), - fragmentation=bool(fragmented and idx >= 1 and a.track_id != track_ids[0]), - track_quality=round(max(quality, 0.0), 3), - ) - evidence[a.annotation_id] = ev - store.put("tracking_evidence", ev) - # Labels without tracks (e.g. false positives) get neutral-low evidence. - for a in annotations: - if a.annotation_id not in evidence: - ev = TrackingEvidence(annotation_id=a.annotation_id, track_quality=0.7 if a.track_id else None) - evidence[a.annotation_id] = ev - store.put("tracking_evidence", ev) - return evidence - - # ------------------------------------------------------------ alerts - - def _refresh_alerts(self, dataset_id: str) -> None: - store = self.store - store.clear("alerts") - m = reporting.quality_metrics(store, dataset_id)["global"] - counters = reporting.counters(store, dataset_id) - qs = self.queue.stats() - - def alert(kind: str, severity: str, message: str, page: str, eid: Optional[str] = None): - store.put("alerts", Alert(alert_id=new_id("alert"), kind=kind, severity=severity, - message=message, evidence_page=page, evidence_id=eid)) - - if self.regression_alert: - alert("model_regression", "critical", "Model regression detected vs baseline", "regression") - if m["anomaly_rate"] > 0.15: - alert("high_anomaly_rate", "warning", - f"Anomaly rate {m['anomaly_rate']:.1%} exceeds 15%", "rare-events") - if m["grader_consensus"] is not None and m["grader_consensus"] < 0.85: - alert("low_consensus", "warning", - f"Mean grader consensus {m['grader_consensus']:.1%} below 85%", "quality") - if m["id_swap_rate"] is not None and m["id_swap_rate"] > 0.05: - alert("high_id_swaps", "warning", - f"ID swap rate {m['id_swap_rate']:.1%} above 5%", "quality") - if m["fragmentation_rate"] is not None and m["fragmentation_rate"] > 0.08: - alert("high_fragmentation", "warning", - f"Track fragmentation {m['fragmentation_rate']:.1%} above 8%", "quality") - total = max(counters["evaluated"], 1) - hitl_rate = counters["flagged"] / total - if hitl_rate > 0.30: - alert("high_hitl_rate", "warning", f"HITL rate {hitl_rate:.1%} above 30%", "review") - ver_rate = counters["verified"] / total - if ver_rate < 0.5 and counters["evaluated"] > 0: - alert("low_verification_rate", "warning", - f"Verification rate {ver_rate:.1%} below 50%", "datasets") - if qs["pending"] > 5000: - alert("queue_backlog", "warning", f"Queue backlog {qs['pending']} messages", "pipeline") - if qs["failed"] > 0: - alert("processing_failure", "critical", f"{qs['failed']} failed queue messages", "pipeline") - if m["safety_critical_recall"] is not None and m["safety_critical_recall"] < 0.8: - alert("safety_recall_low", "critical", - f"Safety-critical recall {m['safety_critical_recall']:.1%} below 80%", "quality") - - # ------------------------------------------------------------ HITL review - - def apply_review(self, task_id: str, action: RelabelingAction, actor: str = "human-reviewer") -> Dict: - """Apply a human action, then ALWAYS re-run validation, grading and - triage gates before any verified/rejected outcome (spec §23).""" - store = self.store - task = store.get("review_tasks", task_id) - if task is None: - raise KeyError(f"Unknown review task {task_id}") - ann = store.get("annotations", task.annotation_id) - frame = store.get("frames", ann.frame_id) - dataset = store.get("datasets", ann.dataset_id) - - store.put("human_reviews", HumanReview( - review_id=new_id("rev"), task_id=task_id, annotation_id=ann.annotation_id, - reviewer=actor, action=action)) - meter = ProcessMeter(store, self.last_run_id or "") - meter.record("hitl", 1) - - act = action.action - if act == "reject": - ann.status = "REJECTED" - store.put("annotations", ann) - task.status = "resolved" - task.resolution = ReviewResolution(action=act, revalidation_passed=None, final_status="REJECTED") - store.put("review_tasks", task) - store.audit("review_rejected", "Annotation", ann.annotation_id, "human rejected label", actor) - store.save() - return {"task": task, "revalidation": None, "message": "Label rejected by human review."} - - if act == "mark_ignore": - ann.status = "REJECTED" - store.put("annotations", ann) - task.status = "resolved" - task.resolution = ReviewResolution(action=act, revalidation_passed=None, final_status="REJECTED") - store.put("review_tasks", task) - store.audit("review_ignored", "Annotation", ann.annotation_id, "marked ignore region", actor) - store.save() - return {"task": task, "revalidation": None, "message": "Marked as ignore; excluded from training."} - - if act == "correct": - if action.corrected_bbox_3d: - ann.bbox_3d = [float(v) for v in action.corrected_bbox_3d] - ann.bbox_2d = synthetic.project_bbox_2d(ann.bbox_3d) - if action.corrected_class: - ann.class_name = action.corrected_class - ann.source = "human_corrected" - self._set_human_reference(frame, ann) - elif act == "approve": - # Human asserts the label is correct: label geometry becomes the - # human-verified reference (the human overrules the vendor GT). - ann.source = "human_approved" - self._set_human_reference(frame, ann) - elif act == "merge_tracks": - if action.merge_with_track_id: - ann.track_id = action.merge_with_track_id - ann.source = "human_corrected" - elif act == "split_track": - ann.track_id = f"{ann.track_id or 'trk'}-split-{task_id[-4:]}" - ann.source = "human_corrected" - store.put("annotations", ann) - - # --- re-validation -> re-grading -> re-triage (gates always re-run) - points = synthetic.frame_points(store, frame) - validation = validate_annotation(store, ann, frame, points=points) - tev = store.get("tracking_evidence", ann.annotation_id) - if act in ("merge_tracks", "split_track") and tev is not None: - tev.id_switch = False - tev.fragmentation = False - tev.track_quality = max(tev.track_quality or 0.7, 0.8) - store.put("tracking_evidence", tev) - grading = grade_annotation(store, ann, frame, - temporal_quality=tev.track_quality if tev else None, - seed=dataset.seed if dataset else 7) - anomaly = self._rescore_anomaly(ann) - decision = triage_annotation( - store, ann, validation=validation, anomaly=anomaly, grading=grading, - tracking=tev, policy=load_policy(), model_regressed=False) - - if decision.status == "AUTO_GRADED": - ann.status = "VERIFIED" - final = "VERIFIED" - else: - ann.status = "FLAGGED" - final = "FLAGGED" - store.put("annotations", ann) - - task.status = "resolved" - task.resolution = ReviewResolution( - action=act, - corrected_bbox_3d=action.corrected_bbox_3d, - corrected_class=action.corrected_class, - revalidation_passed=decision.status == "AUTO_GRADED", - final_status=final, - ) - store.put("review_tasks", task) - # Dataset lineage: corrected by this review batch. - if dataset is not None and task.review_batch: - dataset.lineage.corrected_by_review_batch = task.review_batch - store.put("datasets", dataset) - store.audit(f"review_{act}", "Annotation", ann.annotation_id, - f"re-validation {'PASSED' if final == 'VERIFIED' else 'FAILED: ' + ', '.join(decision.failure_reasons)}", - actor) - self._refresh_alerts(ann.dataset_id) - store.save() - msg = ("Re-validation re-ran all gates: PASSED — label verified." - if final == "VERIFIED" - else f"Re-validation re-ran all gates: still failing {', '.join(decision.failure_reasons)}.") - return {"task": task, "revalidation": None, "message": msg} - - def _set_human_reference(self, frame, ann: Annotation) -> None: - """Human-verified geometry becomes the reference ground truth.""" - store = self.store - gt = next((g for g in frame.gt_boxes if g.gt_id == ann.matched_gt_id), None) - if gt is None: - from sensorflow.evaluation.records import GroundTruthBox - gt = GroundTruthBox( - gt_id=f"{frame.frame_id}-human-{ann.annotation_id[-6:]}", - class_name=ann.class_name, - bbox_3d=list(ann.bbox_3d), - track_instance_id=ann.object_id, - gt_type="HUMAN_VERIFIED_GROUND_TRUTH", - ) - frame.gt_boxes.append(gt) - ann.matched_gt_id = gt.gt_id - else: - gt.class_name = ann.class_name - gt.bbox_3d = list(ann.bbox_3d) - gt.gt_type = "HUMAN_VERIFIED_GROUND_TRUTH" - gt.bbox_2d = synthetic.project_bbox_2d(gt.bbox_3d) - store.put("frames", frame) - - def _rescore_anomaly(self, ann: Annotation) -> Optional[AnomalyDetection]: - """Re-score a corrected label against the fitted ensemble via score - percentiles; falls back to the stored record when no context exists.""" - stored = self.store.get("anomalies", ann.annotation_id) - ctx = self._score_context - if ctx is None: - return stored - try: - X, ids = synthetic.annotation_features(self.store, ann.dataset_id) - if ann.annotation_id not in ids: - return stored - row = X[ids.index(ann.annotation_id)][None, :] - Xs = ctx["scaler"].transform(row) - raw_scores, norm_scores = {}, {} - for det in ctx["detectors"]: - s = float(det.score(Xs)[0]) - raw_scores[det.name] = round(s, 4) - population = ctx["raw"][det.name] - norm_scores[det.name] = round(float((population < s).mean()), 4) - from sensorflow.evaluation.detectors import DEFAULT_WEIGHTS - names = list(norm_scores) - if ctx["strategy"] == "majority_vote": - score = float(np.mean([norm_scores[n] >= ctx["threshold"] for n in names])) - else: - w = np.array([DEFAULT_WEIGHTS.get(n, 1.0) for n in names]) - score = float((np.array([norm_scores[n] for n in names]) * w).sum() / w.sum()) - det_rec = AnomalyDetection( - annotation_id=ann.annotation_id, - score=round(score, 4), - is_anomaly=bool(score >= ctx["threshold"]), - detector_scores=raw_scores, - normalized_scores=norm_scores, - ensemble_strategy=ctx["strategy"], - ensemble_score=round(score, 4), - decision_threshold=ctx["threshold"], - ) - self.store.put("anomalies", det_rec) - return det_rec - except Exception: - return stored - - # ------------------------------------------------------------ training flywheel - - def _active_model_version(self) -> str: - models = sorted(self.store.all("models"), key=lambda m: m.created_at) - return models[-1].model_version if models else "model-v1" - - def start_training(self, dataset_version: str, model_version: Optional[str] = None, - configuration: Optional[Dict] = None, quality_policy: str = "", - training_parameters: Optional[Dict] = None) -> TrainingJob: - store = self.store - dataset = store.get("datasets", dataset_version) - if dataset is None: - for d in store.all("datasets"): - if d.version == dataset_version or d.name == dataset_version: - dataset = d - break - if dataset is None: - raise KeyError(f"Unknown dataset {dataset_version}") - - verified = [a for a in store.where("annotations", dataset_id=dataset.dataset_id) - if a.status == "VERIFIED"] - if not verified: - raise ValueError("No verified labels available for training — run the evaluation pipeline first") - - policy = load_policy() - existing = store.all("models") - next_num = len(existing) + 2 # model-v1 is the implicit generator - new_version = model_version or f"model-v{next_num}" - model_id = new_id("mdl") - - lineage = DatasetLineage( - generated_from_model=dataset.lineage.generated_from_model, - corrected_by_review_batch=dataset.lineage.corrected_by_review_batch, - validated_by_policy=quality_policy or policy.policy_id, - parent_dataset=dataset.dataset_id, - ) - tds = TrainingDataset( - training_dataset_id=new_id("tds"), - source_dataset_id=dataset.dataset_id, - version=f"{dataset.version}-train", - num_verified_labels=len(verified), - lineage=lineage, - ) - store.put("training_datasets", tds) - dataset.lineage.validated_by_policy = policy.policy_id - store.put("datasets", dataset) - - params = training_parameters or {} - job = TrainingJob( - job_id=new_id("job"), - model_id=model_id, - model_version=new_version, - dataset_version=dataset.dataset_id, - total_epochs=int(params.get("epochs", 10)), - status="queued", - quality_policy=quality_policy or policy.policy_id, - configuration=configuration or {}, - lineage=lineage, - ) - job.logs.append(f"[queued] training {new_version} on {len(verified)} verified labels " - f"(dataset {dataset.dataset_id}, policy {job.quality_policy})") - store.put("training_jobs", job) - store.audit("training_started", "TrainingJob", job.job_id, - f"{new_version} on {tds.training_dataset_id} ({len(verified)} verified labels)") - - t = threading.Thread(target=self._train_loop, args=(job.job_id, len(verified)), daemon=True) - self._train_threads[job.job_id] = t - t.start() - return job - - def _train_loop(self, job_id: str, n_verified: int) -> None: - store = self.store - job = store.get("training_jobs", job_id) - meter = ProcessMeter(store, job_id) - job.status = "running" - store.put("training_jobs", job) - rng = np.random.default_rng(abs(hash(job_id)) % (2 ** 32)) - - # Data volume drives the achievable ceiling (flywheel effect). - ceiling = min(0.985, 0.80 + 0.03 * math.log10(max(n_verified, 10))) - prev = self._latest_completed_model() - for epoch in range(1, job.total_epochs + 1): - time.sleep(0.6) - frac = epoch / job.total_epochs - job.epoch = epoch - job.loss = round(float(2.2 * math.exp(-2.5 * frac) + rng.uniform(0.01, 0.04)), 4) - job.rare_recall = round(float(min(ceiling - 0.05, 0.45 + (ceiling - 0.48) * frac + rng.uniform(-0.01, 0.01))), 4) - job.safety_recall = round(float(min(ceiling, 0.55 + (ceiling - 0.55) * frac + rng.uniform(-0.01, 0.01))), 4) - units = meter.record("training", n_verified // job.total_epochs + 1) - job.process_units += units - job.logs.append( - f"[epoch {epoch}/{job.total_epochs}] loss={job.loss:.4f} " - f"rare_recall={job.rare_recall:.3f} safety_recall={job.safety_recall:.3f} pu+={units}") - store.put("training_jobs", job) - - job.status = "completed" - job.logs.append(f"[done] {job.model_version} trained; evaluating vs " - f"{prev.model_version if prev else 'no previous model'}") - store.put("training_jobs", job) - - # Model evaluation + regression/improvement detection. - metrics = ModelEvaluation( - precision=round(min(0.99, ceiling + 0.005), 4), - recall=round(ceiling - 0.01, 4), - f1=round(ceiling - 0.005, 4), - map_3d=round(ceiling - 0.03, 4), - safety_critical_recall=job.safety_recall, - rare_recall=job.rare_recall, - ) - status = "baseline" - if prev is not None and prev.metrics.f1 is not None: - status = "improved" if (metrics.f1 or 0) >= prev.metrics.f1 else "regressed" - compare_runs( - store, - {"precision": metrics.precision, "recall": metrics.recall, - "safety_critical_recall": metrics.safety_critical_recall}, - job.model_version, job.dataset_version, job.job_id, - baseline_metrics={"precision": prev.metrics.precision, "recall": prev.metrics.recall, - "safety_critical_recall": prev.metrics.safety_critical_recall}, - baseline_version=prev.model_version, - ) - model = Model( - model_id=job.model_id, - model_version=job.model_version, - name="synthlab-detector", - trained_on_dataset=job.dataset_version, - status="ready", - metrics=metrics, - regression_status=status, - ) - store.put("models", model) - store.audit("training_completed", "Model", model.model_id, - f"{model.model_version} f1={metrics.f1} status={status}") - store.save() - - def _latest_completed_model(self) -> Optional[Model]: - models = sorted(self.store.all("models"), key=lambda m: m.created_at) - return models[-1] if models else None - - -_PIPELINE: Optional[LabelEvalPipeline] = None -_PIPE_LOCK = threading.Lock() - - -def get_pipeline() -> LabelEvalPipeline: - global _PIPELINE - with _PIPE_LOCK: - if _PIPELINE is None: - _PIPELINE = LabelEvalPipeline() - return _PIPELINE - - -def reset_pipeline(store: Optional[EvalStore] = None) -> LabelEvalPipeline: - global _PIPELINE - with _PIPE_LOCK: - _PIPELINE = LabelEvalPipeline(store=store) - return _PIPELINE diff --git a/hf-space/sensorflow/evaluation/process_units.py b/hf-space/sensorflow/evaluation/process_units.py deleted file mode 100644 index 9fdb86fd4d9f17d2869cadab89f0a408a2587a2c..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/process_units.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Process-unit accounting: deterministic cost metric (not wall-clock). - -Extends the philosophy of sensorflow/metrics/resource_profile.py to the full -evaluation platform: every stage records units; unit economics are derived -from actual state (cost per verified event, per million frames, per dataset). -""" - -from __future__ import annotations - -from typing import Dict, Optional - -from sensorflow.evaluation.records import EvalStore, ProcessUsage, new_id - -# Deterministic per-item complexity factors (units per processed item). -STAGE_FACTORS: Dict[str, float] = { - "ingestion": 0.2, - "label_generation": 1.0, - "anomaly_detection": 0.6, - "rare_event_detection": 0.4, - "rag_retrieval": 0.3, - "vlm": 2.5, - "validation": 0.5, - "grading": 0.8, - "regression_tracking": 0.3, - "hitl": 5.0, - "training": 12.0, -} - - -class ProcessMeter: - """Records process-unit usage into the store, grouped by stage and run.""" - - def __init__(self, store: EvalStore, run_id: str = ""): - self.store = store - self.run_id = run_id - - def record(self, stage: str, items: int, factor: Optional[float] = None) -> int: - f = STAGE_FACTORS.get(stage, 1.0) if factor is None else factor - units = max(1, int(round(items * f))) if items > 0 else 0 - if units > 0: - self.store.put( - "process_usage", - ProcessUsage(usage_id=new_id("pu"), stage=stage, units=units, run_id=self.run_id), - ) - return units - - -def usage_summary(store: EvalStore) -> Dict[str, object]: - by_stage: Dict[str, int] = {} - total = 0 - for u in store.all("process_usage"): - by_stage[u.stage] = by_stage.get(u.stage, 0) + u.units - total += u.units - - verified = len([a for a in store.all("annotations") if a.status == "VERIFIED"]) - rare_verified = len([e for e in store.all("rare_events") if e.verified]) - verified_events = verified + rare_verified - frames = len(store.all("frames")) - training_datasets = len(store.all("training_datasets")) - - return { - "total": total, - "by_stage": by_stage, - "unit_economics": { - "per_verified_event": round(total / verified_events, 2) if verified_events else None, - "per_million_frames": round(total / frames * 1_000_000, 0) if frames else None, - "per_training_dataset": round(total / training_datasets, 2) if training_datasets else None, - }, - } diff --git a/hf-space/sensorflow/evaluation/queue.py b/hf-space/sensorflow/evaluation/queue.py deleted file mode 100644 index fffece2463a74247d6254ea8e8dda68b538d0d49..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/queue.py +++ /dev/null @@ -1,176 +0,0 @@ -"""Queue abstraction between ingestion and evaluation workers. - -The platform is queue-first but transport-agnostic: EventQueue is the interface, -InMemoryEventQueue is the fully-working default, RedisEventQueue is used when a -redis client is importable/reachable, and KafkaEventQueue is an -interface-compatible stub kept for deployment parity (Kafka is not bundled). -""" - -from __future__ import annotations - -import json -import threading -import time -from collections import defaultdict, deque -from typing import Any, Dict, List, Optional - - -class EventQueue: - """Abstract message/job queue keyed by topic.""" - - backend_name = "abstract" - - def publish(self, topic: str, message: Dict[str, Any]) -> None: - raise NotImplementedError - - def consume(self, topic: str, max_messages: int = 100) -> List[Dict[str, Any]]: - raise NotImplementedError - - def ack(self, topic: str, count: int, failed: int = 0) -> None: - raise NotImplementedError - - def stats(self) -> Dict[str, Any]: - raise NotImplementedError - - def reset(self) -> None: - raise NotImplementedError - - -class InMemoryEventQueue(EventQueue): - """Default queue: thread-safe deque per topic with throughput accounting.""" - - backend_name = "in_memory" - - def __init__(self) -> None: - self.lock = threading.RLock() - self.topics: Dict[str, deque] = defaultdict(deque) - self.processing = 0 - self.completed = 0 - self.failed = 0 - self._window: deque = deque(maxlen=500) # (ts, count) for throughput - - def publish(self, topic: str, message: Dict[str, Any]) -> None: - with self.lock: - self.topics[topic].append(message) - - def consume(self, topic: str, max_messages: int = 100) -> List[Dict[str, Any]]: - out = [] - with self.lock: - q = self.topics[topic] - while q and len(out) < max_messages: - out.append(q.popleft()) - self.processing += len(out) - return out - - def ack(self, topic: str, count: int, failed: int = 0) -> None: - with self.lock: - self.processing = max(0, self.processing - count - failed) - self.completed += count - self.failed += failed - self._window.append((time.time(), count)) - - def stats(self) -> Dict[str, Any]: - with self.lock: - depth = {t: len(q) for t, q in self.topics.items() if len(q) > 0} - pending = sum(len(q) for q in self.topics.values()) - now = time.time() - recent = [(ts, c) for ts, c in self._window if now - ts <= 5.0] - throughput = sum(c for _, c in recent) / 5.0 if recent else 0.0 - return { - "backend": self.backend_name, - "pending": pending, - "processing": self.processing, - "completed": self.completed, - "failed": self.failed, - "throughput_per_s": round(throughput, 1), - "depth_by_topic": depth, - } - - def reset(self) -> None: - with self.lock: - self.topics.clear() - self.processing = 0 - self.completed = 0 - self.failed = 0 - self._window.clear() - - -class RedisEventQueue(InMemoryEventQueue): - """Redis-backed queue. Falls back cleanly if redis is unavailable. - - Uses Redis lists per topic; counters are kept locally (single-process dev - deployment). Only constructed when the redis package imports and pings. - """ - - backend_name = "redis" - - def __init__(self, url: str = "redis://localhost:6379/0") -> None: - super().__init__() - import redis # noqa: F401 (raises ImportError when unavailable) - - self.client = redis.Redis.from_url(url, socket_connect_timeout=0.5) - self.client.ping() - self.prefix = "labeleval:queue:" - - def publish(self, topic: str, message: Dict[str, Any]) -> None: - self.client.rpush(self.prefix + topic, json.dumps(message)) - - def consume(self, topic: str, max_messages: int = 100) -> List[Dict[str, Any]]: - out = [] - for _ in range(max_messages): - raw = self.client.lpop(self.prefix + topic) - if raw is None: - break - out.append(json.loads(raw)) - with self.lock: - self.processing += len(out) - return out - - def stats(self) -> Dict[str, Any]: - base = super().stats() - depth = {} - for key in self.client.scan_iter(self.prefix + "*"): - name = key.decode().replace(self.prefix, "") - depth[name] = self.client.llen(key) - base["pending"] = sum(depth.values()) - base["depth_by_topic"] = depth - base["backend"] = self.backend_name - return base - - def reset(self) -> None: - for key in self.client.scan_iter(self.prefix + "*"): - self.client.delete(key) - super().reset() - - -class KafkaEventQueue(EventQueue): - """Interface-compatible Kafka stub. - - Kafka is not bundled with this repo; this class documents the production - transport and raises a clear error if selected without a broker client. - """ - - backend_name = "kafka" - - def __init__(self, bootstrap_servers: str = "localhost:9092") -> None: - self.bootstrap_servers = bootstrap_servers - raise RuntimeError( - "KafkaEventQueue is a deployment stub: install/configure a Kafka client " - "and broker, or use InMemoryEventQueue (default) / RedisEventQueue." - ) - - -def make_queue(preferred: str = "auto") -> EventQueue: - """Build the best available queue. Never hard-codes to Kafka.""" - if preferred in ("auto", "redis"): - try: - return RedisEventQueue() - except Exception: - if preferred == "redis": - pass # fall through to in-memory with a working queue - if preferred == "kafka": - try: - return KafkaEventQueue() - except Exception: - pass - return InMemoryEventQueue() diff --git a/hf-space/sensorflow/evaluation/rare_events.py b/hf-space/sensorflow/evaluation/rare_events.py deleted file mode 100644 index 201c51271c180a5ab896d5088cd1396b76ec3e48..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/rare_events.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Rare event detection & scenario mining. - -Finds rare/high-value safety scenarios (spec §15): near collisions, extreme -TTC/PET, VRU interactions, unusual object behavior, severe occlusions, sensor -failures, nighttime glare, adverse weather, unusual trajectories, unexpected -road geometry. Produces RareEvent records with rarity/anomaly scores and -per-sensor evidence. Reuses the SSAM-style TTC/PET surrogate-safety logic. -""" - -from __future__ import annotations - -import math -from collections import defaultdict -from typing import Dict, List, Optional - -import numpy as np - -from sensorflow.evaluation.records import EvalStore, Frame, RareEvent, new_id - -SEVERITY_BY_TYPE = { - "near_collision": "critical", - "extreme_ttc": "high", - "extreme_pet": "high", - "vru_interaction": "high", - "sensor_failure": "high", - "unusual_object_behavior": "medium", - "unusual_trajectory": "medium", - "severe_occlusion": "medium", - "nighttime_glare": "low", - "adverse_weather": "low", - "unexpected_road_geometry": "medium", -} - - -def _frame_ttc(frame: Frame) -> Optional[float]: - """Minimum time-to-collision across GT objects approaching the ego path.""" - ttcs = [] - for gt in frame.gt_boxes: - x, y = gt.bbox_3d[0], gt.bbox_3d[1] - vx, vy = gt.velocity[0], gt.velocity[1] - if abs(y) < 2.5 and vx < -0.3: - ttcs.append(x / (-vx)) - # Laterally converging object crossing the ego corridor. - if vy != 0 and abs(y) > 1.0 and np.sign(vy) == -np.sign(y): - t_cross = abs(y) / abs(vy) - x_at_cross = x + vx * t_cross - if 0 < x_at_cross < 12 and t_cross < 6.0: - ttcs.append(t_cross) - return min(ttcs) if ttcs else None - - -def _frame_pet(frame: Frame) -> Optional[float]: - """Post-encroachment-time surrogate: time gap between an object leaving the - ego corridor and the ego arriving at that point.""" - pets = [] - ego_speed = max(frame.ego_pose.speed_mps, 0.1) - for gt in frame.gt_boxes: - x, y = gt.bbox_3d[0], gt.bbox_3d[1] - vy = gt.velocity[1] - if abs(y) < 1.5 and abs(vy) > 0.5: # object currently inside corridor, exiting laterally - t_exit = (1.5 - abs(y)) / abs(vy) - t_ego_arrival = x / ego_speed - pet = t_ego_arrival - t_exit - if pet > 0: - pets.append(pet) - return min(pets) if pets else None - - -def detect_rare_events( - store: EvalStore, - dataset_id: str, - anomaly_scores_by_frame: Optional[Dict[str, float]] = None, -) -> List[RareEvent]: - """Mine sequences for rare scenarios; group contiguous frames per type.""" - frames = sorted(store.where("frames", dataset_id=dataset_id), key=lambda f: f.index) - anomaly_scores_by_frame = anomaly_scores_by_frame or {} - - # Derive per-frame scenario hits (tags + kinematic measures). - hits: List[Dict] = [] - for fr in frames: - frame_hits: Dict[str, Dict[str, str]] = {} - ttc = _frame_ttc(fr) - pet = _frame_pet(fr) - if ttc is not None and ttc < 3.0: - frame_hits["extreme_ttc"] = {"lidar": f"TTC={ttc:.2f}s from range-rate", "telemetry": "converging range"} - if ttc < 1.8: - frame_hits["near_collision"] = {"lidar": f"TTC={ttc:.2f}s inside ego corridor", - "camera": "object crossing ego path"} - if pet is not None and pet < 1.5: - frame_hits["extreme_pet"] = {"telemetry": f"PET={pet:.2f}s corridor encroachment"} - for tag in fr.scenario_tags: - if tag == "vru_interaction": - frame_hits["vru_interaction"] = {"camera": "VRU within 15m of ego path", "lidar": "VRU point cluster"} - elif tag == "sensor_failure": - frame_hits["sensor_failure"] = {"lidar": f"only {fr.num_lidar_points} returns (expected >600)"} - elif tag == "severe_occlusion": - frame_hits["severe_occlusion"] = {"lidar": "object with <10 supporting points"} - elif tag == "night_glare": - frame_hits["nighttime_glare"] = {"camera": "night scene, glare-prone exposure"} - elif tag == "adverse_weather": - frame_hits["adverse_weather"] = {"camera": "rain droplets on lens", "lidar": "rain-scatter noise"} - # Unusual trajectory / behavior: wrong-way or erratic objects. - for gt in fr.gt_boxes: - if gt.velocity[0] < -20: # closing far faster than ego speed => oncoming in our lane region - frame_hits["unusual_trajectory"] = {"lidar": f"object closing at {-gt.velocity[0]:.1f} m/s", - "map": "movement against traffic direction"} - frame_hits["unusual_object_behavior"] = {"telemetry": "erratic heading vs lane geometry"} - if frame_hits: - hits.append({"frame": fr, "hits": frame_hits}) - - # Scenario frequency -> rarity. - type_counts: Dict[str, int] = defaultdict(int) - for h in hits: - for t in h["hits"]: - type_counts[t] += 1 - total_frames = max(len(frames), 1) - - # Group contiguous frames of same sequence+type into one event. - events: List[RareEvent] = [] - open_events: Dict[str, Dict] = {} - for h in hits: - fr: Frame = h["frame"] - for t, evidence in h["hits"].items(): - key = f"{fr.sequence_id}:{t}" - if key in open_events and fr.index - open_events[key]["last_index"] <= 2: - open_events[key]["frames"].append(fr.frame_id) - open_events[key]["last_index"] = fr.index - open_events[key]["evidence"].update(evidence) - else: - if key in open_events: - events.append(_finalize(store, dataset_id, open_events.pop(key), - type_counts, total_frames, anomaly_scores_by_frame)) - open_events[key] = {"type": t, "frames": [fr.frame_id], "last_index": fr.index, - "evidence": dict(evidence)} - for ev in open_events.values(): - events.append(_finalize(store, dataset_id, ev, type_counts, total_frames, anomaly_scores_by_frame)) - - for e in events: - store.put("rare_events", e) - store.audit("rare_events_detected", "Dataset", dataset_id, f"{len(events)} events") - return events - - -def _finalize( - store: EvalStore, - dataset_id: str, - ev: Dict, - type_counts: Dict[str, int], - total_frames: int, - anomaly_scores_by_frame: Dict[str, float], -) -> RareEvent: - t = ev["type"] - freq = type_counts[t] / total_frames - rarity = float(np.clip(1.0 - freq * 4.0, 0.05, 1.0)) - scores = [anomaly_scores_by_frame.get(fid, 0.0) for fid in ev["frames"]] - anomaly = float(np.mean(scores)) if scores else 0.0 - return RareEvent( - event_id=new_id("evt"), - dataset_id=dataset_id, - scenario_type=t, - severity=SEVERITY_BY_TYPE.get(t, "medium"), - rarity_score=round(rarity, 4), - anomaly_score=round(anomaly, 4), - confidence=round(float(np.clip(0.55 + 0.4 * rarity, 0, 0.99)), 3), - evidence_frames=ev["frames"], - sensor_evidence=ev["evidence"], - description=f"{t.replace('_', ' ').title()} spanning {len(ev['frames'])} frame(s)", - ) diff --git a/hf-space/sensorflow/evaluation/records.py b/hf-space/sensorflow/evaluation/records.py deleted file mode 100644 index 417b85a1cdec593a9812af34a639daa957dba8b6..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/records.py +++ /dev/null @@ -1,611 +0,0 @@ -"""Data model + JSON-backed store for the label evaluation platform. - -Entities (spec §37): Dataset, Scene, Sequence, Frame, Sensor, Object, Annotation, -Track, Scenario, RareEvent, AnomalyDetection, ValidationResult, GroundTruth, -Grader, GraderComparison, RegressionResult, Benchmark, BenchmarkResult, -TriageDecision, ReviewTask, HumanReview, RelabelingAction, TrainingDataset, -TrainingJob, Model, ModelEvaluation, ProcessUsage, AuditEvent. -""" - -from __future__ import annotations - -import json -import threading -import uuid -from datetime import datetime, timezone -from pathlib import Path -from typing import Any, Dict, List, Literal, Optional - -from pydantic import BaseModel, Field - -STORE_DIR = Path("runs/labeleval") - -GroundTruthType = Literal[ - "PSEUDO_GROUND_TRUTH", - "VENDOR_GROUND_TRUTH", - "HUMAN_VERIFIED_GROUND_TRUTH", - "GOLD_STANDARD", -] - -TriageStatus = Literal["AUTO_GRADED", "FLAGGED", "VERIFIED", "REJECTED", "PENDING"] - -FAILURE_REASONS = [ - "LOW_IOU", - "POSITION_ERROR", - "ORIENTATION_ERROR", - "INSUFFICIENT_POINT_SUPPORT", - "ANOMALY", - "GRADER_DISAGREEMENT", - "TRACK_FRAGMENTATION", - "ID_SWITCH", - "MODEL_REGRESSION", - "LOW_CONFIDENCE", - "SENSOR_DISAGREEMENT", -] - -CLASSES = ["pedestrian", "cyclist", "vehicle", "motorcycle", "truck"] -SAFETY_CRITICAL_CLASSES = {"pedestrian", "cyclist", "motorcycle"} - -SCENARIO_TYPES = [ - "near_collision", - "extreme_ttc", - "extreme_pet", - "vru_interaction", - "unusual_object_behavior", - "severe_occlusion", - "sensor_failure", - "nighttime_glare", - "adverse_weather", - "unusual_trajectory", - "unexpected_road_geometry", -] - - -def now_iso() -> str: - return datetime.now(timezone.utc).isoformat() - - -def new_id(prefix: str) -> str: - return f"{prefix}-{uuid.uuid4().hex[:10]}" - - -# ------------------------------------------------------------------ sensor data - - -class EgoState(BaseModel): - x: float = 0.0 - y: float = 0.0 - z: float = 0.0 - yaw: float = 0.0 - speed_mps: float = 0.0 - - -class GroundTruthBox(BaseModel): - gt_id: str - class_name: str - bbox_3d: List[float] # [x, y, z, l, w, h, yaw] - bbox_2d: Optional[List[float]] = None - track_instance_id: str = "" - gt_type: GroundTruthType = "VENDOR_GROUND_TRUTH" - velocity: List[float] = Field(default_factory=lambda: [0.0, 0.0]) - - -class Frame(BaseModel): - frame_id: str - dataset_id: str - scene_id: str - sequence_id: str - index: int = 0 - timestamp_us: int = 0 - ego_pose: EgoState = Field(default_factory=EgoState) - num_lidar_points: int = 0 - camera_width: int = 800 - camera_height: int = 450 - gt_boxes: List[GroundTruthBox] = Field(default_factory=list) - scenario_tags: List[str] = Field(default_factory=list) - weather: str = "clear" - time_of_day: str = "day" - - -class Sequence(BaseModel): - sequence_id: str - scene_id: str - dataset_id: str - frame_ids: List[str] = Field(default_factory=list) - - -class Scene(BaseModel): - scene_id: str - dataset_id: str - sequence_ids: List[str] = Field(default_factory=list) - description: str = "" - - -class DatasetLineage(BaseModel): - generated_from_model: Optional[str] = None - corrected_by_review_batch: Optional[str] = None - validated_by_policy: Optional[str] = None - parent_dataset: Optional[str] = None - - -class Dataset(BaseModel): - dataset_id: str - name: str - version: str = "v1" - created_at: str = Field(default_factory=now_iso) - seed: int = 7 - num_scenes: int = 0 - num_sequences: int = 0 - num_frames: int = 0 - num_annotations: int = 0 - gt_type: Optional[GroundTruthType] = "VENDOR_GROUND_TRUTH" - gt_coverage: float = 0.0 - lineage: DatasetLineage = Field(default_factory=DatasetLineage) - status: str = "created" - generation_params: Dict[str, Any] = Field(default_factory=dict) - - -# ------------------------------------------------------------------ labels - - -class Annotation(BaseModel): - """Auto-generated label: a hypothesis requiring evaluation. Never assumed correct.""" - - annotation_id: str - dataset_id: str - frame_id: str - object_id: str - class_name: str - confidence: float = 0.0 - bbox_2d: Optional[List[float]] = None # [x, y, w, h] image px - bbox_3d: Optional[List[float]] = None # [x, y, z, l, w, h, yaw] - mask: Optional[str] = None - track_id: Optional[str] = None - model: str = "synthlab-detector" - model_version: str = "model-v1" - source: str = "auto" - status: TriageStatus = "PENDING" - matched_gt_id: Optional[str] = None - injected_errors: List[str] = Field(default_factory=list) - - -class Track(BaseModel): - track_id: str - dataset_id: str - class_name: str = "vehicle" - annotation_ids: List[str] = Field(default_factory=list) - frame_ids: List[str] = Field(default_factory=list) - - -class Scenario(BaseModel): - scenario_id: str - dataset_id: str - scenario_type: str - frame_ids: List[str] = Field(default_factory=list) - description: str = "" - - -class RareEvent(BaseModel): - event_id: str - dataset_id: str - scenario_type: str - severity: Literal["low", "medium", "high", "critical"] = "medium" - rarity_score: float = 0.0 - anomaly_score: float = 0.0 - confidence: float = 0.0 - evidence_frames: List[str] = Field(default_factory=list) - sensor_evidence: Dict[str, str] = Field(default_factory=dict) - description: str = "" - verified: bool = False - - -# ------------------------------------------------------------------ evaluation evidence - - -class AnomalyDetection(BaseModel): - annotation_id: str - score: float = 0.0 - is_anomaly: bool = False - detector_scores: Dict[str, float] = Field(default_factory=dict) - normalized_scores: Dict[str, float] = Field(default_factory=dict) - ensemble_strategy: str = "weighted_average" - ensemble_score: float = 0.0 - decision_threshold: float = 0.7 - - -class CheckLine(BaseModel): - gate: str - actual: Any = None - threshold: Any = None - passed: bool = True - applicable: bool = True - - -class ValidationResult(BaseModel): - annotation_id: str - passed: bool = True - checks: List[CheckLine] = Field(default_factory=list) - iou_3d: Optional[float] = None - position_error: Optional[float] = None - orientation_error_deg: Optional[float] = None - dimension_error: Optional[float] = None - point_density: Optional[float] = None - point_in_box_ratio: Optional[float] = None - ground_contact_error: Optional[float] = None - sensor_consistent: bool = True - - -class GraderComparison(BaseModel): - annotation_id: str - grader_count: int = 0 - graders: List[str] = Field(default_factory=list) - class_votes: Dict[str, str] = Field(default_factory=dict) - class_agreement: Optional[float] = None - spatial_agreement: Optional[float] = None - temporal_agreement: Optional[float] = None - consensus: Optional[float] = None - disagreement_types: List[str] = Field(default_factory=list) - kappa_stats: Dict[str, float] = Field(default_factory=dict) - - -class TrackingEvidence(BaseModel): - annotation_id: str - id_switch: bool = False - fragmentation: bool = False - track_quality: Optional[float] = None - - -class TriageDecision(BaseModel): - decision_id: str - annotation_id: str - status: TriageStatus = "PENDING" - failure_reasons: List[str] = Field(default_factory=list) - primary_failure_reason: Optional[str] = None - gate_lines: List[CheckLine] = Field(default_factory=list) - policy_id: str = "" - policy_values: Dict[str, float] = Field(default_factory=dict) - decided_at: str = Field(default_factory=now_iso) - - -# ------------------------------------------------------------------ review - - -class RelabelingAction(BaseModel): - action: str - corrected_bbox_3d: Optional[List[float]] = None - corrected_class: Optional[str] = None - merge_with_track_id: Optional[str] = None - note: Optional[str] = None - - -class ReviewResolution(BaseModel): - action: str - corrected_bbox_3d: Optional[List[float]] = None - corrected_class: Optional[str] = None - revalidation_passed: Optional[bool] = None - final_status: Optional[TriageStatus] = None - reviewed_at: str = Field(default_factory=now_iso) - - -class ReviewTask(BaseModel): - task_id: str - annotation_id: str - frame_id: str - dataset_id: str - failure_reasons: List[str] = Field(default_factory=list) - primary_failure_reason: Optional[str] = None - status: Literal["open", "in_review", "resolved"] = "open" - created_at: str = Field(default_factory=now_iso) - review_batch: str = "" - resolution: Optional[ReviewResolution] = None - - -class HumanReview(BaseModel): - review_id: str - task_id: str - annotation_id: str - reviewer: str = "human-reviewer" - action: RelabelingAction - created_at: str = Field(default_factory=now_iso) - - -# ------------------------------------------------------------------ regression / benchmark - - -class RegressionMetricDelta(BaseModel): - metric: str - baseline: float - current: float - delta: float - tolerance: float - regressed: bool - - -class RegressionResult(BaseModel): - result_id: str - model_version: str - baseline_version: Optional[str] = None - dataset_version: str = "" - run_id: str = "" - date: str = Field(default_factory=now_iso) - regression_detected: bool = False - affected_classes: List[str] = Field(default_factory=list) - affected_scenarios: List[str] = Field(default_factory=list) - deltas: List[RegressionMetricDelta] = Field(default_factory=list) - kinds: List[str] = Field(default_factory=list) - - -class BenchmarkResult(BaseModel): - technique: str - precision: float = 0.0 - recall: float = 0.0 - rare_recall: float = 0.0 - f1: float = 0.0 - box_error_3d: float = 0.0 - id_swap_rate: float = 0.0 - consensus: float = 0.0 - process_units: int = 0 - fp_rate: float = 0.0 - - -class Benchmark(BaseModel): - benchmark_id: str - dataset_id: str = "" - created_at: str = Field(default_factory=now_iso) - rows: List[BenchmarkResult] = Field(default_factory=list) - highlights: Dict[str, str] = Field(default_factory=dict) - - -# ------------------------------------------------------------------ training / models - - -class TrainingDataset(BaseModel): - training_dataset_id: str - source_dataset_id: str - version: str - num_verified_labels: int = 0 - lineage: DatasetLineage = Field(default_factory=DatasetLineage) - created_at: str = Field(default_factory=now_iso) - - -class TrainingJob(BaseModel): - job_id: str - model_id: str - model_version: str - dataset_version: str - status: Literal["queued", "running", "completed", "failed", "stopped"] = "queued" - epoch: int = 0 - total_epochs: int = 10 - loss: float = 0.0 - rare_recall: float = 0.0 - safety_recall: float = 0.0 - process_units: int = 0 - logs: List[str] = Field(default_factory=list) - started_at: str = Field(default_factory=now_iso) - quality_policy: str = "" - configuration: Dict[str, Any] = Field(default_factory=dict) - lineage: DatasetLineage = Field(default_factory=DatasetLineage) - - -class ModelEvaluation(BaseModel): - precision: Optional[float] = None - recall: Optional[float] = None - f1: Optional[float] = None - map_3d: Optional[float] = None - safety_critical_recall: Optional[float] = None - rare_recall: Optional[float] = None - - -class Model(BaseModel): - model_id: str - model_version: str - name: str = "synthlab-detector" - trained_on_dataset: Optional[str] = None - created_at: str = Field(default_factory=now_iso) - status: str = "ready" - metrics: ModelEvaluation = Field(default_factory=ModelEvaluation) - regression_status: Literal["baseline", "improved", "regressed", "unknown"] = "unknown" - - -# ------------------------------------------------------------------ ops - - -class ProcessUsage(BaseModel): - usage_id: str - stage: str - units: int = 0 - run_id: str = "" - created_at: str = Field(default_factory=now_iso) - - -class AuditEvent(BaseModel): - event_id: str - timestamp: str = Field(default_factory=now_iso) - actor: str = "system" - action: str = "" - entity_type: str = "" - entity_id: str = "" - detail: str = "" - - -class Alert(BaseModel): - alert_id: str - kind: str - severity: Literal["info", "warning", "critical"] = "warning" - message: str = "" - evidence_page: str = "overview" - evidence_id: Optional[str] = None - created_at: str = Field(default_factory=now_iso) - - -# ------------------------------------------------------------------ store - -_COLLECTIONS = { - "datasets": Dataset, - "scenes": Scene, - "sequences": Sequence, - "frames": Frame, - "annotations": Annotation, - "tracks": Track, - "scenarios": Scenario, - "rare_events": RareEvent, - "anomalies": AnomalyDetection, - "validations": ValidationResult, - "grader_comparisons": GraderComparison, - "tracking_evidence": TrackingEvidence, - "triage_decisions": TriageDecision, - "review_tasks": ReviewTask, - "human_reviews": HumanReview, - "regressions": RegressionResult, - "benchmarks": Benchmark, - "training_datasets": TrainingDataset, - "training_jobs": TrainingJob, - "models": Model, - "process_usage": ProcessUsage, - "audit_events": AuditEvent, - "alerts": Alert, -} - -_KEY_FIELDS = { - "datasets": "dataset_id", - "scenes": "scene_id", - "sequences": "sequence_id", - "frames": "frame_id", - "annotations": "annotation_id", - "tracks": "track_id", - "scenarios": "scenario_id", - "rare_events": "event_id", - "anomalies": "annotation_id", - "validations": "annotation_id", - "grader_comparisons": "annotation_id", - "tracking_evidence": "annotation_id", - "triage_decisions": "annotation_id", # latest decision per annotation - "review_tasks": "task_id", - "human_reviews": "review_id", - "regressions": "result_id", - "benchmarks": "benchmark_id", - "training_datasets": "training_dataset_id", - "training_jobs": "job_id", - "models": "model_id", - "process_usage": "usage_id", - "audit_events": "event_id", - "alerts": "alert_id", -} - - -class EvalStore: - """Thread-safe in-memory store with JSON persistence. - - Determinism and explainability matter more than storage tech (spec). - """ - - def __init__(self, base_dir: Path = STORE_DIR): - self.base_dir = Path(base_dir) - self.lock = threading.RLock() - self.collections: Dict[str, Dict[str, BaseModel]] = {c: {} for c in _COLLECTIONS} - self.meta: Dict[str, Any] = {} - self._load() - - # -- persistence - - def _path(self) -> Path: - return self.base_dir / "store.json" - - def _load(self) -> None: - path = self._path() - if not path.exists(): - return - try: - with open(path) as f: - raw = json.load(f) - except Exception: - return - with self.lock: - for name, model_cls in _COLLECTIONS.items(): - items = raw.get(name, {}) - parsed: Dict[str, BaseModel] = {} - for key, payload in items.items(): - try: - parsed[key] = model_cls.model_validate(payload) - except Exception: - continue - self.collections[name] = parsed - self.meta = raw.get("_meta", {}) - - def save(self) -> None: - with self.lock: - payload = { - name: {k: v.model_dump() for k, v in items.items()} - for name, items in self.collections.items() - } - payload["_meta"] = self.meta - self.base_dir.mkdir(parents=True, exist_ok=True) - tmp = self._path().with_suffix(".tmp") - with open(tmp, "w") as f: - json.dump(payload, f) - tmp.replace(self._path()) - - # -- generic access - - def put(self, collection: str, item: BaseModel) -> None: - key = getattr(item, _KEY_FIELDS[collection]) - with self.lock: - self.collections[collection][key] = item - - def get(self, collection: str, key: str) -> Optional[Any]: - with self.lock: - return self.collections[collection].get(key) - - def all(self, collection: str) -> List[Any]: - with self.lock: - return list(self.collections[collection].values()) - - def where(self, collection: str, **filters: Any) -> List[Any]: - with self.lock: - items = list(self.collections[collection].values()) - out = [] - for it in items: - if all(getattr(it, k, None) == v for k, v in filters.items()): - out.append(it) - return out - - def delete(self, collection: str, key: str) -> None: - with self.lock: - self.collections[collection].pop(key, None) - - def clear(self, collection: str) -> None: - with self.lock: - self.collections[collection] = {} - - # -- audit helper - - def audit(self, action: str, entity_type: str, entity_id: str, detail: str = "", actor: str = "system") -> None: - evt = AuditEvent( - event_id=new_id("audit"), - actor=actor, - action=action, - entity_type=entity_type, - entity_id=entity_id, - detail=detail, - ) - self.put("audit_events", evt) - - -_STORE: Optional[EvalStore] = None -_STORE_LOCK = threading.Lock() - - -def get_store() -> EvalStore: - global _STORE - with _STORE_LOCK: - if _STORE is None: - _STORE = EvalStore() - return _STORE - - -def reset_store(base_dir: Optional[Path] = None) -> EvalStore: - """Used by tests to get a fresh isolated store.""" - global _STORE - with _STORE_LOCK: - _STORE = EvalStore(base_dir or STORE_DIR) - return _STORE diff --git a/hf-space/sensorflow/evaluation/regression.py b/hf-space/sensorflow/evaluation/regression.py deleted file mode 100644 index da2d1fcbdffe9393f0eb9c8fd8774aa8114b7020..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/regression.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Performance regression tracking across model/dataset versions and runs. - -Flags (spec §18): -- performance regression: precision/recall/safety-recall below baseline-tolerance -- tracking regression: IDF1 down, ID switches up, fragmentation up -- annotation regression: 3D IoU down, position/orientation error up -""" - -from __future__ import annotations - -from typing import Dict, List, Optional - -from sensorflow.evaluation.records import ( - EvalStore, - RegressionMetricDelta, - RegressionResult, - new_id, -) - -# metric -> (tolerance, direction). "up" means higher is better. -METRIC_SPECS: Dict[str, Dict] = { - "precision": {"tolerance": 0.03, "better": "up", "kind": "performance"}, - "recall": {"tolerance": 0.03, "better": "up", "kind": "performance"}, - "safety_critical_recall": {"tolerance": 0.02, "better": "up", "kind": "performance"}, - "idf1": {"tolerance": 0.05, "better": "up", "kind": "tracking"}, - "id_swap_rate": {"tolerance": 0.02, "better": "down", "kind": "tracking"}, - "fragmentation_rate": {"tolerance": 0.03, "better": "down", "kind": "tracking"}, - "mean_iou_3d": {"tolerance": 0.03, "better": "up", "kind": "annotation"}, - "mean_position_error": {"tolerance": 0.10, "better": "down", "kind": "annotation"}, - "mean_orientation_error_deg": {"tolerance": 3.0, "better": "down", "kind": "annotation"}, -} - - -def compare_runs( - store: EvalStore, - current_metrics: Dict[str, Optional[float]], - model_version: str, - dataset_version: str, - run_id: str, - baseline_metrics: Optional[Dict[str, Optional[float]]] = None, - baseline_version: Optional[str] = None, - affected_classes: Optional[List[str]] = None, - affected_scenarios: Optional[List[str]] = None, -) -> RegressionResult: - """Compare current run metrics vs a baseline run; persist RegressionResult.""" - deltas: List[RegressionMetricDelta] = [] - kinds: List[str] = [] - regressed_any = False - - if baseline_metrics: - for metric, spec in METRIC_SPECS.items(): - base = baseline_metrics.get(metric) - cur = current_metrics.get(metric) - if base is None or cur is None: - continue - delta = cur - base - if spec["better"] == "up": - regressed = delta < -spec["tolerance"] - else: - regressed = delta > spec["tolerance"] - deltas.append(RegressionMetricDelta( - metric=metric, - baseline=round(float(base), 4), - current=round(float(cur), 4), - delta=round(float(delta), 4), - tolerance=spec["tolerance"], - regressed=regressed, - )) - if regressed: - regressed_any = True - if spec["kind"] not in kinds: - kinds.append(spec["kind"]) - - result = RegressionResult( - result_id=new_id("reg"), - model_version=model_version, - baseline_version=baseline_version, - dataset_version=dataset_version, - run_id=run_id, - regression_detected=regressed_any, - affected_classes=affected_classes or [], - affected_scenarios=affected_scenarios or [], - deltas=deltas, - kinds=kinds, - ) - store.put("regressions", result) - if regressed_any: - store.audit("regression_detected", "RegressionResult", result.result_id, - f"{model_version} vs {baseline_version}: kinds={kinds} classes={result.affected_classes}") - return result - - -def find_affected_classes( - current_per_class: Dict[str, Dict[str, Optional[float]]], - baseline_per_class: Dict[str, Dict[str, Optional[float]]], -) -> List[str]: - """Classes whose precision or recall regressed beyond tolerance.""" - affected = [] - for cls, cur in current_per_class.items(): - base = baseline_per_class.get(cls) - if not base: - continue - for metric in ("precision", "recall"): - b, c = base.get(metric), cur.get(metric) - if b is not None and c is not None and (c - b) < -METRIC_SPECS[metric]["tolerance"]: - affected.append(cls) - break - return affected diff --git a/hf-space/sensorflow/evaluation/reporting.py b/hf-space/sensorflow/evaluation/reporting.py deleted file mode 100644 index 4de33809f7f36df6a9389c64c0c340651ed91aa1..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/reporting.py +++ /dev/null @@ -1,706 +0,0 @@ -"""Aggregated reporting: quality metrics, funnel, groups, haystack, benchmark. - -Ground-truth integrity (spec §41): precision/recall are only computed when a -reference exists; model confidence is never presented as precision; recall is -never inferred from anomaly scores. -""" - -from __future__ import annotations - -import math -from collections import defaultdict -from typing import Dict, List, Optional - -import numpy as np - -from sensorflow.evaluation import synthetic -from sensorflow.evaluation.process_units import usage_summary -from sensorflow.evaluation.records import ( - Benchmark, - BenchmarkResult, - EvalStore, - SAFETY_CRITICAL_CLASSES, - new_id, -) - -TP_IOU = 0.5 - - -# ------------------------------------------------------------------ matching - - -def _match_stats(store: EvalStore, dataset_id: str, frame_ids: Optional[set] = None, - class_name: Optional[str] = None) -> Dict: - """TP/FP/FN over annotations vs reference GT (only where reference exists).""" - frames = store.where("frames", dataset_id=dataset_id) - if frame_ids is not None: - frames = [f for f in frames if f.frame_id in frame_ids] - fid_set = {f.frame_id for f in frames} - - anns = [a for a in store.where("annotations", dataset_id=dataset_id) - if a.frame_id in fid_set and a.status != "REJECTED"] - if class_name: - anns = [a for a in anns if a.class_name == class_name] - - gt_total = 0 - gt_hit: Dict[str, bool] = {} - for f in frames: - for g in f.gt_boxes: - if class_name and g.class_name != class_name: - continue - gt_total += 1 - gt_hit[g.gt_id] = False - - tp = fp = 0 - for a in anns: - v = store.get("validations", a.annotation_id) - iou = v.iou_3d if v else None - if a.matched_gt_id and a.matched_gt_id in gt_hit and iou is not None and iou >= TP_IOU: - if not gt_hit[a.matched_gt_id]: - gt_hit[a.matched_gt_id] = True - tp += 1 - else: - fp += 1 - else: - fp += 1 - fn = sum(1 for hit in gt_hit.values() if not hit) - return {"tp": tp, "fp": fp, "fn": fn, "gt_total": gt_total, "n_annotations": len(anns)} - - -def _prf(stats: Dict) -> Dict[str, Optional[float]]: - tp, fp, fn = stats["tp"], stats["fp"], stats["fn"] - if stats["gt_total"] == 0 and tp + fp == 0: - return {"precision": None, "recall": None, "f1": None} - precision = tp / (tp + fp) if (tp + fp) > 0 else None - recall = tp / (tp + fn) if (tp + fn) > 0 else None - f1 = None - if precision is not None and recall is not None and (precision + recall) > 0: - f1 = 2 * precision * recall / (precision + recall) - r = lambda v: round(v, 4) if v is not None else None - return {"precision": r(precision), "recall": r(recall), "f1": r(f1)} - - -# ------------------------------------------------------------------ metrics - - -def quality_metrics(store: EvalStore, dataset_id: Optional[str]) -> Dict: - dataset = store.get("datasets", dataset_id) if dataset_id else None - empty = { - "dataset_id": dataset_id, "gt_available": False, "gt_type": None, "gt_coverage": 0.0, - "global": {k: None for k in ( - "precision", "recall", "f1", "map_3d", "safety_critical_recall", "mean_iou_3d", - "mean_position_error", "mean_orientation_error_deg", "idf1", "id_swap_rate", - "fragmentation_rate", "grader_consensus")} | {"anomaly_rate": 0.0}, - "per_class": [], "per_scenario": [], - } - if dataset is None: - return empty - - anns = [a for a in store.where("annotations", dataset_id=dataset_id) if a.status != "REJECTED"] - if not anns: - return empty - - gt_available = dataset.gt_coverage > 0 - - stats = _match_stats(store, dataset_id) - prf = _prf(stats) if gt_available else {"precision": None, "recall": None, "f1": None} - - # Safety-critical recall (reference required). - saf_tp = saf_fn = 0 - for cls in SAFETY_CRITICAL_CLASSES: - s = _match_stats(store, dataset_id, class_name=cls) - saf_tp += s["tp"] - saf_fn += s["fn"] - safety_recall = round(saf_tp / (saf_tp + saf_fn), 4) if gt_available and (saf_tp + saf_fn) > 0 else None - - # Geometry means over matched labels. - ious, pos_errs, yaw_errs = [], [], [] - anomaly_flags, consensus_vals = [], [] - switches = frags = tracked = 0 - for a in anns: - v = store.get("validations", a.annotation_id) - if v: - if v.iou_3d is not None: - ious.append(v.iou_3d) - if v.position_error is not None: - pos_errs.append(v.position_error) - if v.orientation_error_deg is not None: - yaw_errs.append(v.orientation_error_deg) - an = store.get("anomalies", a.annotation_id) - if an: - anomaly_flags.append(1.0 if an.is_anomaly else 0.0) - g = store.get("grader_comparisons", a.annotation_id) - if g and g.consensus is not None: - consensus_vals.append(g.consensus) - t = store.get("tracking_evidence", a.annotation_id) - if t and a.matched_gt_id: - tracked += 1 - switches += 1 if t.id_switch else 0 - frags += 1 if t.fragmentation else 0 - - id_swap_rate = round(switches / tracked, 4) if tracked else None - frag_rate = round(frags / tracked, 4) if tracked else None - # IDF1 approximation from identity error rates (full IDF1 needs global - # bipartite matching; this monotone surrogate is used consistently). - idf1 = None - if id_swap_rate is not None and frag_rate is not None: - idf1 = round(max(0.0, 1.0 - id_swap_rate - 0.5 * frag_rate), 4) - - map_3d = _map_3d(store, dataset_id) if gt_available else None - - glob = { - "precision": prf["precision"], - "recall": prf["recall"], - "f1": prf["f1"], - "map_3d": map_3d, - "safety_critical_recall": safety_recall, - "mean_iou_3d": round(float(np.mean(ious)), 4) if ious else None, - "mean_position_error": round(float(np.mean(pos_errs)), 4) if pos_errs else None, - "mean_orientation_error_deg": round(float(np.mean(yaw_errs)), 4) if yaw_errs else None, - "idf1": idf1, - "id_swap_rate": id_swap_rate, - "fragmentation_rate": frag_rate, - "anomaly_rate": round(float(np.mean(anomaly_flags)), 4) if anomaly_flags else 0.0, - "grader_consensus": round(float(np.mean(consensus_vals)), 4) if consensus_vals else None, - } - - per_class = [] - for cls in ("pedestrian", "cyclist", "vehicle", "motorcycle", "truck"): - s = _match_stats(store, dataset_id, class_name=cls) - if s["gt_total"] == 0 and s["n_annotations"] == 0: - continue - p = _prf(s) if gt_available else {"precision": None, "recall": None, "f1": None} - per_class.append({"class_name": cls, **p, "support": s["gt_total"]}) - - per_scenario = [] - for scn in store.where("scenarios", dataset_id=dataset_id): - fids = set(scn.frame_ids) - s = _match_stats(store, dataset_id, frame_ids=fids) - p = _prf(s) if gt_available else {"precision": None, "recall": None, "f1": None} - per_scenario.append({"scenario": scn.scenario_type, **p, "support": s["gt_total"]}) - - return { - "dataset_id": dataset_id, - "gt_available": gt_available, - "gt_type": dataset.gt_type if gt_available else None, - "gt_coverage": dataset.gt_coverage, - "global": glob, - "per_class": per_class, - "per_scenario": per_scenario, - } - - -def _map_3d(store: EvalStore, dataset_id: str, max_frames: int = 40) -> Optional[float]: - """Mean AP over IoU thresholds {0.5, 0.7}, averaged over sampled frames.""" - from sensorflow.metrics.perception_3d import compute_map_mar - - frames = sorted(store.where("frames", dataset_id=dataset_id), key=lambda f: f.index) - if not frames: - return None - step = max(1, len(frames) // max_frames) - sampled = frames[::step] - anns_by_frame: Dict[str, List] = defaultdict(list) - for a in store.where("annotations", dataset_id=dataset_id): - if a.status != "REJECTED" and a.bbox_3d: - anns_by_frame[a.frame_id].append(a.bbox_3d) - aps = [] - for f in sampled: - gts = [g.bbox_3d for g in f.gt_boxes] - preds = anns_by_frame.get(f.frame_id, []) - if not gts and not preds: - continue - m = compute_map_mar(preds, gts) - aps.append(m["map_3d"]) - return round(float(np.mean(aps)), 4) if aps else None - - -# ------------------------------------------------------------------ counters / funnel - - -def counters(store: EvalStore, dataset_id: Optional[str] = None) -> Dict[str, int]: - anns = store.where("annotations", dataset_id=dataset_id) if dataset_id else store.all("annotations") - frames = store.where("frames", dataset_id=dataset_id) if dataset_id else store.all("frames") - evaluated = sum(1 for a in anns if store.get("triage_decisions", a.annotation_id) is not None) - decisions = [store.get("triage_decisions", a.annotation_id) for a in anns] - auto_graded = sum(1 for d in decisions if d is not None and d.status == "AUTO_GRADED") - open_tasks = [t for t in store.all("review_tasks") - if t.status != "resolved" and (dataset_id is None or t.dataset_id == dataset_id)] - events = store.where("rare_events", dataset_id=dataset_id) if dataset_id else store.all("rare_events") - return { - "frames_processed": len(frames), - "auto_labeled": len(anns), - "evaluated": evaluated, - "auto_graded": auto_graded, - "flagged": sum(1 for a in anns if a.status == "FLAGGED"), - "in_hitl": len(open_tasks), - "verified": sum(1 for a in anns if a.status == "VERIFIED"), - "rejected": sum(1 for a in anns if a.status == "REJECTED"), - "rare_events": len(events), - } - - -def funnel(store: EvalStore, dataset_id: Optional[str] = None) -> Dict: - c = counters(store, dataset_id) - anns = store.where("annotations", dataset_id=dataset_id) if dataset_id else store.all("annotations") - validated = sum(1 for a in anns if store.get("validations", a.annotation_id) is not None) - tds = store.all("training_datasets") - if dataset_id: - tds = [t for t in tds if t.source_dataset_id == dataset_id] - in_training = sum(t.num_verified_labels for t in tds) - - tasks = [t for t in store.all("review_tasks") if dataset_id is None or t.dataset_id == dataset_id] - resolved = [t for t in tasks if t.status == "resolved"] - relabeled = [t for t in resolved if t.resolution and t.resolution.action in - ("correct", "approve", "merge_tracks", "split_track")] - revalidated = [t for t in relabeled if t.resolution.revalidation_passed is not None] - re_verified = [t for t in relabeled if t.resolution.final_status == "VERIFIED"] - - base = max(c["auto_labeled"], 1) - - def stage(name: str, count: int) -> Dict: - return {"stage": name, "count": count, "pct_of_input": round(100.0 * count / base, 1)} - - main = [ - stage("RAW SENSOR DATA (frames)", c["frames_processed"]), - stage("AUTO-GENERATED LABELS", c["auto_labeled"]), - stage("EVALUATED", c["evaluated"]), - stage("QUALITY VALIDATION", validated), - stage("AUTO-GRADED", c["auto_graded"]), - stage("VERIFIED", c["verified"]), - stage("TRAINING", in_training), - ] - side = [ - stage("FLAGGED", c["flagged"] + len(resolved)), - stage("HITL REVIEW", len(tasks)), - stage("RE-LABELING", len(relabeled)), - stage("RE-VALIDATION", len(revalidated)), - stage("VERIFIED (post-HITL)", len(re_verified)), - ] - return {"main_path": main, "side_path": side} - - -# ------------------------------------------------------------------ groups - - -GROUP_NAMES = ["verified", "non_verified", "hitl", "rejected"] - - -def _group_members(store: EvalStore, dataset_id: str) -> Dict[str, List]: - anns = store.where("annotations", dataset_id=dataset_id) - open_hitl = {t.annotation_id for t in store.all("review_tasks") - if t.dataset_id == dataset_id and t.status != "resolved"} - groups: Dict[str, List] = {g: [] for g in GROUP_NAMES} - for a in anns: - if a.status == "VERIFIED": - groups["verified"].append(a) - elif a.status == "REJECTED": - groups["rejected"].append(a) - elif a.annotation_id in open_hitl: - groups["hitl"].append(a) - else: - groups["non_verified"].append(a) - return groups - - -def quality_groups(store: EvalStore, dataset_id: str) -> Dict: - groups = _group_members(store, dataset_id) - total = sum(len(v) for v in groups.values()) - out = [] - for name in GROUP_NAMES: - members = groups[name] - out.append({ - "group_id": f"{dataset_id}:{name}", - "dataset_id": dataset_id, - "name": name, - "count": len(members), - "pct": round(100.0 * len(members) / max(total, 1), 1), - }) - return { - "dataset_id": dataset_id, - "total": total, - "verification_rate": round(len(groups["verified"]) / max(total, 1), 4), - "groups": out, - } - - -def group_detail(store: EvalStore, group_id: str) -> Optional[Dict]: - if ":" not in group_id: - return None - dataset_id, name = group_id.rsplit(":", 1) - groups = _group_members(store, dataset_id) - if name not in groups: - return None - members = groups[name] - total = sum(len(v) for v in groups.values()) - - ious, cons, anom, tq = [], [], [], [] - reason_counts: Dict[str, int] = defaultdict(int) - tp = fp = 0 - for a in members: - v = store.get("validations", a.annotation_id) - if v and v.iou_3d is not None: - ious.append(v.iou_3d) - if v.iou_3d >= TP_IOU: - tp += 1 - else: - fp += 1 - elif v: - fp += 1 - g = store.get("grader_comparisons", a.annotation_id) - if g and g.consensus is not None: - cons.append(g.consensus) - an = store.get("anomalies", a.annotation_id) - if an: - anom.append(an.score) - t = store.get("tracking_evidence", a.annotation_id) - if t and t.track_quality is not None: - tq.append(t.track_quality) - d = store.get("triage_decisions", a.annotation_id) - if d: - for r in d.failure_reasons: - reason_counts[r] += 1 - - dataset = store.get("datasets", dataset_id) - gt_available = dataset is not None and dataset.gt_coverage > 0 - precision = round(tp / (tp + fp), 4) if gt_available and (tp + fp) > 0 else None - # Group-level recall vs all reference objects claimed by this group's labels. - matched_gt = {a.matched_gt_id for a in members if a.matched_gt_id} - recall = round(tp / max(len(matched_gt), 1), 4) if gt_available and matched_gt else None - f1 = None - if precision is not None and recall is not None and precision + recall > 0: - f1 = round(2 * precision * recall / (precision + recall), 4) - - mean = lambda arr: round(float(np.mean(arr)), 4) if arr else None - return { - "group_id": group_id, - "dataset_id": dataset_id, - "name": name, - "count": len(members), - "pct": round(100.0 * len(members) / max(total, 1), 1), - "precision": precision, - "recall": recall, - "f1": f1, - "mean_iou_3d": mean(ious), - "mean_consensus": mean(cons), - "mean_anomaly_score": mean(anom), - "tracking_quality": mean(tq), - "failure_reason_counts": dict(reason_counts), - "annotation_ids": [a.annotation_id for a in members][:500], - } - - -# ------------------------------------------------------------------ haystack - - -def haystack(store: EvalStore, dataset_id: Optional[str] = None) -> List[Dict]: - """2D embedding of all observations: normals, anomalies, rare events, - detected FP/FN and verified labels. Clicking a point opens its evidence.""" - if dataset_id is None: - datasets = sorted(store.all("datasets"), key=lambda d: d.created_at) - if not datasets: - return [] - dataset_id = datasets[-1].dataset_id - - X, ids = synthetic.annotation_features(store, dataset_id) - if len(ids) == 0: - return [] - Xs = (X - X.mean(axis=0)) / (X.std(axis=0) + 1e-9) - _, _, vt = np.linalg.svd(Xs, full_matrices=False) - proj = Xs @ vt[:2].T - # scale to [2, 98] - lo, hi = proj.min(axis=0), proj.max(axis=0) - span = np.maximum(hi - lo, 1e-9) - coords = 2 + 96 * (proj - lo) / span - - anns = {a.annotation_id: a for a in store.where("annotations", dataset_id=dataset_id)} - rare_frames: Dict[str, str] = {} - for e in store.where("rare_events", dataset_id=dataset_id): - for fid in e.evidence_frames: - rare_frames[fid] = e.event_id - - points: List[Dict] = [] - frame_positions: Dict[str, List[np.ndarray]] = defaultdict(list) - for j, aid in enumerate(ids): - a = anns[aid] - an = store.get("anomalies", aid) - d = store.get("triage_decisions", aid) - v = store.get("validations", aid) - frame_positions[a.frame_id].append(coords[j]) - if a.matched_gt_id is None and a.status in ("FLAGGED", "REJECTED"): - cat = "false_positive" - elif v and v.iou_3d is not None and v.iou_3d < TP_IOU and a.status in ("FLAGGED", "REJECTED"): - cat = "false_positive" - elif an and an.is_anomaly: - cat = "anomaly" - elif a.status == "VERIFIED" and a.source != "auto": - cat = "verified" - elif a.status == "VERIFIED": - cat = "normal" - elif d is not None and d.status == "FLAGGED": - cat = "anomaly" if (an and an.score > 0.7) else "normal" - else: - cat = "normal" - points.append({ - "id": aid, - "x": round(float(coords[j][0]), 2), - "y": round(float(coords[j][1]), 2), - "category": cat, - "anomaly_score": round(float(an.score), 4) if an else 0.0, - "class_name": a.class_name, - "frame_id": a.frame_id, - "kind": "annotation", - }) - - # Missed objects (false negatives): reference GT never matched by a label. - matched = {a.matched_gt_id for a in anns.values() if a.matched_gt_id} - rng = np.random.default_rng(13) - for f in store.where("frames", dataset_id=dataset_id): - for g in f.gt_boxes: - if g.gt_id in matched: - continue - base = np.mean(frame_positions[f.frame_id], axis=0) if frame_positions.get(f.frame_id) else np.array([50.0, 50.0]) - jitter = rng.normal(0, 3, 2) - points.append({ - "id": f"fn-{g.gt_id}", - "x": round(float(np.clip(base[0] + jitter[0], 1, 99)), 2), - "y": round(float(np.clip(base[1] + jitter[1], 1, 99)), 2), - "category": "false_negative", - "anomaly_score": 0.0, - "class_name": g.class_name, - "frame_id": f.frame_id, - "kind": "annotation", - }) - - # Rare events as first-class points near their evidence cluster. - for e in store.where("rare_events", dataset_id=dataset_id): - pos = [np.mean(frame_positions[fid], axis=0) for fid in e.evidence_frames if frame_positions.get(fid)] - base = np.mean(pos, axis=0) if pos else np.array([90.0, 90.0]) - points.append({ - "id": e.event_id, - "x": round(float(np.clip(base[0] + 1.5, 1, 99)), 2), - "y": round(float(np.clip(base[1] + 1.5, 1, 99)), 2), - "category": "rare_event", - "anomaly_score": e.anomaly_score, - "class_name": e.scenario_type, - "frame_id": e.evidence_frames[0] if e.evidence_frames else "", - "kind": "rare_event", - }) - return points - - -# ------------------------------------------------------------------ evaluation record - - -def evaluation_record(store: EvalStore, annotation_id: str) -> Optional[Dict]: - a = store.get("annotations", annotation_id) - if a is None: - return None - frame = store.get("frames", a.frame_id) - gt = None - if frame is not None and a.matched_gt_id: - gt = next((g for g in frame.gt_boxes if g.gt_id == a.matched_gt_id), None) - v = store.get("validations", annotation_id) - an = store.get("anomalies", annotation_id) - g = store.get("grader_comparisons", annotation_id) - t = store.get("tracking_evidence", annotation_id) - d = store.get("triage_decisions", annotation_id) - - return { - "annotation_id": annotation_id, - "dataset_id": a.dataset_id, - "frame_id": a.frame_id, - "object_class": a.class_name, - "model_version": a.model_version, - "ground_truth_id": a.matched_gt_id, - "ground_truth_type": gt.gt_type if gt else None, - "detection": {"confidence": a.confidence}, - "geometry": { - "iou_3d": v.iou_3d if v else None, - "position_error": v.position_error if v else None, - "orientation_error_deg": v.orientation_error_deg if v else None, - "dimension_error": v.dimension_error if v else None, - "point_density": v.point_density if v else None, - "point_in_box_ratio": v.point_in_box_ratio if v else None, - "ground_contact_error": v.ground_contact_error if v else None, - }, - "tracking": { - "id_switch": t.id_switch if t else False, - "fragmentation": t.fragmentation if t else False, - "track_quality": t.track_quality if t else None, - }, - "anomaly": { - "score": an.score if an else 0.0, - "is_anomaly": an.is_anomaly if an else False, - "detector_scores": an.detector_scores if an else {}, - "normalized_scores": an.normalized_scores if an else {}, - "ensemble_strategy": an.ensemble_strategy if an else "weighted_average", - "decision_threshold": an.decision_threshold if an else 0.9, - }, - "grading": { - "consensus": g.consensus if g else None, - "class_agreement": g.class_agreement if g else None, - "spatial_agreement": g.spatial_agreement if g else None, - "temporal_agreement": g.temporal_agreement if g else None, - "grader_count": g.grader_count if g else 0, - "disagreement_types": g.disagreement_types if g else [], - }, - "validation": { - "passed": v.passed if v else False, - "checks": [c.model_dump() for c in v.checks] if v else [], - }, - "decision": d.model_dump() if d else None, - "injected_errors": a.injected_errors, - } - - -# ------------------------------------------------------------------ frame summary - - -def frame_summary(store: EvalStore, frame_id: str, max_points: int = 900) -> Optional[Dict]: - f = store.get("frames", frame_id) - if f is None: - return None - pts = synthetic.frame_points(store, f) - if len(pts) > max_points: - idx = np.linspace(0, len(pts) - 1, max_points).astype(int) - pts = pts[idx] - anns = store.where("annotations", frame_id=frame_id) - return { - "frame_id": f.frame_id, - "sequence_id": f.sequence_id, - "scene_id": f.scene_id, - "timestamp_us": f.timestamp_us, - "ego_pose": {"x": f.ego_pose.x, "y": f.ego_pose.y, "z": f.ego_pose.z, - "yaw": f.ego_pose.yaw, "speed_mps": f.ego_pose.speed_mps}, - "num_lidar_points": f.num_lidar_points, - "camera": {"width": f.camera_width, "height": f.camera_height}, - "annotations": [a.model_dump() for a in anns], - "gt_boxes": [{ - "gt_id": g.gt_id, "class_name": g.class_name, "bbox_3d": g.bbox_3d, - "bbox_2d": g.bbox_2d, "gt_type": g.gt_type, - } for g in f.gt_boxes], - "lidar_points_bev": [[round(float(p[0]), 2), round(float(p[1]), 2), round(float(p[2]), 2)] for p in pts], - } - - -# ------------------------------------------------------------------ benchmark of techniques - - -TECH_PU_FACTORS = { - "knn": 0.4, "lof": 0.5, "isolation_forest": 0.7, "ocsvm": 0.9, "dbscan": 0.5, - "autoencoder": 1.4, "vae": 1.1, "gan": 1.0, "few_shot": 0.8, - "ensemble_majority_vote": 3.2, "ensemble_weighted_average": 3.2, "ensemble_meta_classifier": 3.6, -} - - -def benchmark_techniques(store: EvalStore, dataset_id: Optional[str], anomaly_config: Dict) -> Benchmark: - """Comparative benchmark: each technique + the three ensembles, evaluated - against the injected-defect ground truth of the synthetic dataset.""" - from sensorflow.evaluation.detectors import AnomalyEnsemble, normalize_scores - - if dataset_id is None: - datasets = sorted(store.all("datasets"), key=lambda d: d.created_at) - if not datasets: - raise ValueError("No dataset available to benchmark") - dataset_id = datasets[-1].dataset_id - dataset = store.get("datasets", dataset_id) - - X, ids = synthetic.annotation_features(store, dataset_id) - if len(ids) < 20: - raise ValueError("Not enough labels to benchmark") - anns = {a.annotation_id: a for a in store.where("annotations", dataset_id=dataset_id)} - y = np.array([1.0 if anns[i].injected_errors else 0.0 for i in ids]) - rare_fids = {fid for e in store.where("rare_events", dataset_id=dataset_id) for fid in e.evidence_frames} - in_rare = np.array([1.0 if anns[i].frame_id in rare_fids else 0.0 for i in ids]) - id_sw = np.array([1.0 if "ID_SWITCH" in anns[i].injected_errors else 0.0 for i in ids]) - pos_err = np.array([ - (store.get("validations", i).position_error or 0.0) if store.get("validations", i) else 0.0 - for i in ids]) - consensus = np.array([ - (store.get("grader_comparisons", i).consensus or 0.0) if store.get("grader_comparisons", i) else 0.0 - for i in ids]) - - cfg = dict(anomaly_config) - # Enable everything for a complete benchmark table. - full_cfg = { - **cfg, - "detectors": {k: {**v, "enabled": True} for k, v in cfg.get("detectors", {}).items()}, - "deep": {**cfg.get("deep", {}), - "autoencoder": {**cfg.get("deep", {}).get("autoencoder", {}), "enabled": True}, - "vae": {**cfg.get("deep", {}).get("vae", {}), "enabled": True}, - "gan": {"enabled": True}}, - "advanced": {**cfg.get("advanced", {}), - "few_shot": {**cfg.get("advanced", {}).get("few_shot", {}), "enabled": True}}, - } - ensemble = AnomalyEnsemble(full_cfg, seed=dataset.seed if dataset else 7) - _, raw, norm = ensemble.run(X, supervision=y) - - contamination = max(y.mean(), 0.02) - k = int(round(contamination * len(y))) - - def evaluate(scores: np.ndarray, technique: str) -> BenchmarkResult: - order = np.argsort(-scores) - detected = np.zeros(len(scores)) - detected[order[:k]] = 1.0 - tp = float((detected * y).sum()) - fp = float((detected * (1 - y)).sum()) - fn = float(((1 - detected) * y).sum()) - tn = float(((1 - detected) * (1 - y)).sum()) - precision = tp / (tp + fp) if tp + fp else 0.0 - recall = tp / (tp + fn) if tp + fn else 0.0 - f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 - rare_mask = (in_rare * y) > 0 - rare_recall = float((detected[rare_mask]).mean()) if rare_mask.any() else recall - missed = detected == 0 - box_err = float(pos_err[missed & (y > 0)].mean()) if (missed & (y > 0)).any() else 0.0 - residual_swaps = float(id_sw[missed].sum() / max(id_sw.sum(), 1)) - cons = float(consensus[missed].mean()) if missed.any() else 0.0 - pu = int(len(ids) * TECH_PU_FACTORS.get(technique, 1.0)) - return BenchmarkResult( - technique=technique, - precision=round(precision, 4), recall=round(recall, 4), - rare_recall=round(rare_recall, 4), f1=round(f1, 4), - box_error_3d=round(box_err, 4), id_swap_rate=round(residual_swaps, 4), - consensus=round(cons, 4), process_units=pu, - fp_rate=round(fp / (fp + tn) if fp + tn else 0.0, 4), - ) - - rows = [evaluate(normalize_scores(raw[name]), name) for name in raw] - for strategy in ("majority_vote", "weighted_average", "meta_classifier"): - e2 = AnomalyEnsemble({**full_cfg, "advanced": {**full_cfg["advanced"], "ensemble_strategy": strategy}}, - seed=dataset.seed if dataset else 7) - s, _, _ = e2.run(X, supervision=y) - rows.append(evaluate(np.asarray(s), f"ensemble_{strategy}")) - - highlights = { - "best_rare_recall": max(rows, key=lambda r: r.rare_recall).technique, - "best_safety_recall": max(rows, key=lambda r: r.recall).technique, - "lowest_fp_rate": min(rows, key=lambda r: r.fp_rate).technique, - "lowest_process_units": min(rows, key=lambda r: r.process_units).technique, - "lowest_tracking_error": min(rows, key=lambda r: r.id_swap_rate).technique, - } - bench = Benchmark(benchmark_id=new_id("bench"), dataset_id=dataset_id, rows=rows, highlights=highlights) - store.put("benchmarks", bench) - store.audit("benchmark_run", "Benchmark", bench.benchmark_id, f"{len(rows)} techniques on {dataset_id}") - return bench - - -# ------------------------------------------------------------------ overview - - -def overview(store: EvalStore, pipeline) -> Dict: - dataset_id = pipeline.active_dataset - c = counters(store, dataset_id) - m = quality_metrics(store, dataset_id)["global"] - total = max(c["auto_labeled"], 1) - pu = usage_summary(store) - models = sorted(store.all("models"), key=lambda x: x.created_at) - return { - "counters": c, - "metrics": m, - "verification_rate": round(c["verified"] / total, 4), - "automation_rate": round(c["auto_graded"] / total, 4), - "process_units_total": pu["total"], - "active_dataset": dataset_id, - "active_model": models[-1].model_version if models else "model-v1", - } diff --git a/hf-space/sensorflow/evaluation/synthetic.py b/hf-space/sensorflow/evaluation/synthetic.py deleted file mode 100644 index f7e44e553eecc9f06610eb32fc84eeb5e98abc78..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/synthetic.py +++ /dev/null @@ -1,542 +0,0 @@ -"""Synthetic sensor data + auto-label generator with known error injection. - -Generates a deterministic dataset (scenes -> sequences -> frames -> GT boxes, -ego motion, LiDAR point clouds) and then auto-generated labels (hypotheses) -with injected, recorded defects so every evaluation subsystem can be verified: -FALSE_POSITIVE, FALSE_NEGATIVE, BAD_3D_BOX, WRONG_ORIENTATION, WRONG_POSITION, -ID_SWITCH, TRACK_FRAGMENTATION, LOW_POINT_DENSITY, SENSOR_DISAGREEMENT, -GRADER_DISAGREEMENT, LOW_CONFIDENCE, RARE_EVENT scenarios, MODEL_REGRESSION. - -All geometry is in the ego frame per frame: x forward (m), y left (m), z up (m). -Ground plane at z=0. Point clouds are re-derived deterministically from the -dataset seed, so nothing heavy is persisted. -""" - -from __future__ import annotations - -import math -from typing import Dict, List, Optional, Tuple - -import numpy as np - -from sensorflow.evaluation.records import ( - Annotation, - Dataset, - DatasetLineage, - EvalStore, - Frame, - GroundTruthBox, - EgoState, - Scene, - Scenario, - Sequence, - Track, - new_id, -) - -CLASS_DIMS = { - "vehicle": (4.5, 1.9, 1.6), - "truck": (8.0, 2.5, 3.1), - "pedestrian": (0.7, 0.7, 1.75), - "cyclist": (1.8, 0.6, 1.7), - "motorcycle": (2.2, 0.8, 1.4), -} -CLASS_POOL = ["vehicle", "vehicle", "vehicle", "truck", "pedestrian", "pedestrian", "cyclist", "motorcycle"] - -CAM_W, CAM_H, CAM_F, CAM_Z = 800, 450, 500.0, 1.5 - -ERROR_TYPES = [ - "FALSE_POSITIVE", - "FALSE_NEGATIVE", - "BAD_3D_BOX", - "WRONG_ORIENTATION", - "WRONG_POSITION", - "ID_SWITCH", - "TRACK_FRAGMENTATION", - "LOW_POINT_DENSITY", - "SENSOR_DISAGREEMENT", - "GRADER_DISAGREEMENT", - "LOW_CONFIDENCE", -] - - -def project_bbox_2d(bbox_3d: List[float]) -> Optional[List[float]]: - x, y, z, l, w, h, _ = bbox_3d - if x < 3.0: - return None - u = CAM_W / 2 - (y / x) * CAM_F - v = CAM_H / 2 + ((CAM_Z - z) / x) * CAM_F - bw = max(6.0, CAM_F * max(l, w) / x) - bh = max(6.0, CAM_F * h / x) - if u + bw / 2 < 0 or u - bw / 2 > CAM_W: - return None - return [round(u - bw / 2, 1), round(v - bh / 2, 1), round(bw, 1), round(bh, 1)] - - -# ------------------------------------------------------------------ dataset generation - - -def generate_dataset( - store: EvalStore, - name: str = "synthetic-perception", - num_sequences: int = 6, - frames_per_sequence: int = 30, - seed: int = 7, - version: str = "v1", - generated_from_model: Optional[str] = "model-v1", -) -> Dataset: - rng = np.random.default_rng(seed) - dataset_id = new_id("ds") - dataset = Dataset( - dataset_id=dataset_id, - name=name, - version=version, - seed=seed, - gt_type="VENDOR_GROUND_TRUTH", - lineage=DatasetLineage(generated_from_model=generated_from_model), - generation_params={ - "num_sequences": num_sequences, - "frames_per_sequence": frames_per_sequence, - "sparse_gt": [], - "injected": {}, - }, - status="generated", - ) - - sparse_gt: List[List[str]] = [] - scenario_frames: Dict[str, List[str]] = {} - - num_scenes = max(1, num_sequences // 2) - scenes = [] - for si in range(num_scenes): - scene = Scene(scene_id=f"{dataset_id}-scene-{si}", dataset_id=dataset_id, description=f"Synthetic scene {si}") - scenes.append(scene) - - frame_count = 0 - for qi in range(num_sequences): - scene = scenes[qi % num_scenes] - seq_id = f"{dataset_id}-seq-{qi}" - sequence = Sequence(sequence_id=seq_id, scene_id=scene.scene_id, dataset_id=dataset_id) - scene.sequence_ids.append(seq_id) - - # Sequence-level conditions (rare-event scenario diversity). - weather = "rain" if qi % 5 == 2 else "clear" - time_of_day = "night" if qi % 5 == 1 else "day" - - # Object population: world-frame kinematics; ego drives +x at 10 m/s. - num_objects = int(rng.integers(5, 9)) - objects = [] - for oi in range(num_objects): - cls = CLASS_POOL[int(rng.integers(0, len(CLASS_POOL)))] - l, w, h = CLASS_DIMS[cls] - x0 = float(rng.uniform(12, 65)) - y0 = float(rng.uniform(-12, 12)) - heading = float(rng.uniform(-math.pi, math.pi)) if cls == "pedestrian" else float(rng.choice([0.0, math.pi, math.pi / 2, -math.pi / 2])) - speed = {"pedestrian": 1.4, "cyclist": 4.5, "motorcycle": 9.0, "vehicle": 8.0, "truck": 7.0}[cls] * float(rng.uniform(0.6, 1.2)) - objects.append({ - "iid": f"{seq_id}-obj-{oi}", - "cls": cls, - "dims": (l, w, h), - "x": x0, "y": y0, - "vx": speed * math.cos(heading), - "vy": speed * math.sin(heading), - "yaw": heading, - }) - - # Rare-event injections per sequence. - if qi % 3 == 0: - # Near-collision: pedestrian cutting across the ego path, converging. - objects.append({ - "iid": f"{seq_id}-obj-nearmiss", - "cls": "pedestrian", - "dims": CLASS_DIMS["pedestrian"], - "x": 40.0, "y": 8.0, "vx": 0.0, "vy": -2.2, "yaw": -math.pi / 2, - }) - if qi % 3 == 1: - # Unusual trajectory: vehicle moving erratically against traffic. - objects.append({ - "iid": f"{seq_id}-obj-erratic", - "cls": "vehicle", - "dims": CLASS_DIMS["vehicle"], - "x": 55.0, "y": -3.0, "vx": -14.0, "vy": 1.5, "yaw": math.pi, - }) - - dt = 0.1 - ego_speed = 10.0 - sensor_failure_frames = set() - if qi % 3 == 2 and frames_per_sequence >= 10: - sensor_failure_frames = {frames_per_sequence // 2, frames_per_sequence // 2 + 1} - - for fi in range(frames_per_sequence): - frame_id = f"{dataset_id}-f{frame_count:05d}" - ego_x = ego_speed * dt * fi - tags: List[str] = [] - if time_of_day == "night": - tags.append("night_glare") - if weather == "rain": - tags.append("adverse_weather") - if fi in sensor_failure_frames: - tags.append("sensor_failure") - - gt_boxes: List[GroundTruthBox] = [] - for obj in objects: - wx = obj["x"] + obj["vx"] * dt * fi - wy = obj["y"] + obj["vy"] * dt * fi - rel_x = wx - ego_x - rel_y = wy - if rel_x < 1.0 or rel_x > 80 or abs(rel_y) > 25: - continue - l, w, h = obj["dims"] - gt = GroundTruthBox( - gt_id=f"{frame_id}-{obj['iid']}", - class_name=obj["cls"], - bbox_3d=[round(rel_x, 3), round(rel_y, 3), round(h / 2, 3), l, w, h, round(obj["yaw"], 4)], - track_instance_id=obj["iid"], - gt_type="VENDOR_GROUND_TRUTH", - velocity=[round(obj["vx"] - ego_speed, 3), round(obj["vy"], 3)], - ) - gt.bbox_2d = project_bbox_2d(gt.bbox_3d) - gt_boxes.append(gt) - - # TTC tagging for approaching objects near the ego path. - rvx = obj["vx"] - ego_speed - if rvx < -0.5 and abs(rel_y) < 2.5: - ttc = rel_x / (-rvx) - if ttc < 3.0: - tags.append("extreme_ttc") - if ttc < 1.8: - tags.append("near_collision") - if obj["cls"] in ("pedestrian", "cyclist") and rel_x < 15 and abs(rel_y) < 5: - tags.append("vru_interaction") - - # Occlusion: mark ~8% of frames; their first GT object is point-sparse. - occluded = rng.random() < 0.08 - if occluded and gt_boxes: - tags.append("severe_occlusion") - sparse_gt.append([frame_id, gt_boxes[0].gt_id]) - - frame = Frame( - frame_id=frame_id, - dataset_id=dataset_id, - scene_id=scene.scene_id, - sequence_id=seq_id, - index=frame_count, - timestamp_us=int(fi * dt * 1e6) + qi * 10_000_000, - ego_pose=EgoState(x=round(ego_x, 2), speed_mps=ego_speed), - gt_boxes=gt_boxes, - scenario_tags=sorted(set(tags)), - weather=weather, - time_of_day=time_of_day, - num_lidar_points=0, # filled below deterministically - camera_width=CAM_W, - camera_height=CAM_H, - ) - frame.num_lidar_points = _expected_point_count(dataset, frame, sparse_gt) - sequence.frame_ids.append(frame_id) - store.put("frames", frame) - frame_count += 1 - - store.put("sequences", sequence) - - for scene in scenes: - store.put("scenes", scene) - - dataset.generation_params["sparse_gt"] = sparse_gt - dataset.num_scenes = num_scenes - dataset.num_sequences = num_sequences - dataset.num_frames = frame_count - # Frame-level reference coverage; replaced by annotation-level coverage - # once labels are generated. - frames_with_gt = sum(1 for f in store.where("frames", dataset_id=dataset_id) if f.gt_boxes) - dataset.gt_coverage = round(frames_with_gt / max(frame_count, 1), 4) - store.put("datasets", dataset) - - # Scenario records (mined tags -> Scenario entities). - tag_frames: Dict[str, List[str]] = {} - for fr in store.where("frames", dataset_id=dataset_id): - for t in fr.scenario_tags: - tag_frames.setdefault(t, []).append(fr.frame_id) - for tag, fids in tag_frames.items(): - store.put("scenarios", Scenario( - scenario_id=f"{dataset_id}-scn-{tag}", - dataset_id=dataset_id, - scenario_type=tag, - frame_ids=sorted(fids), - description=f"Frames exhibiting {tag.replace('_', ' ')}", - )) - scenario_frames.update(tag_frames) - - store.audit("dataset_generated", "Dataset", dataset_id, f"{frame_count} frames, {num_sequences} sequences") - return dataset - - -def _expected_point_count(dataset: Dataset, frame: Frame, sparse_gt: List[List[str]]) -> int: - if "sensor_failure" in frame.scenario_tags: - return 60 - n = 600 - sparse_ids = {g for f, g in sparse_gt if f == frame.frame_id} - for gt in frame.gt_boxes: - n += 8 if gt.gt_id in sparse_ids else _object_point_count(gt) - return n - - -def _object_point_count(gt: GroundTruthBox) -> int: - x, y = gt.bbox_3d[0], gt.bbox_3d[1] - dist = max(3.0, math.hypot(x, y)) - l, w, h = gt.bbox_3d[3], gt.bbox_3d[4], gt.bbox_3d[5] - base = l * w * h * 100 - return int(max(6, min(800, base * (12.0 / dist) ** 1.3))) - - -def frame_points(store: EvalStore, frame: Frame) -> np.ndarray: - """Deterministically re-derive the LiDAR point cloud for a frame (ego frame).""" - dataset = store.get("datasets", frame.dataset_id) - seed = (dataset.seed if dataset else 0) * 100003 + frame.index - rng = np.random.default_rng(seed) - sparse = set() - if dataset: - sparse = {g for f, g in dataset.generation_params.get("sparse_gt", []) if f == frame.frame_id} - - if "sensor_failure" in frame.scenario_tags: - ground = rng.uniform([0, -20, -0.05], [70, 20, 0.05], size=(60, 3)) - return ground.astype(np.float32) - - clouds = [rng.uniform([0, -20, -0.05], [70, 20, 0.05], size=(600, 3))] - for gt in frame.gt_boxes: - n = 8 if gt.gt_id in sparse else _object_point_count(gt) - x, y, z, l, w, h, yaw = gt.bbox_3d - local = rng.uniform([-l / 2, -w / 2, -h / 2], [l / 2, w / 2, h / 2], size=(n, 3)) - c, s = math.cos(yaw), math.sin(yaw) - rot = np.array([[c, -s, 0], [s, c, 0], [0, 0, 1]]) - pts = local @ rot.T + np.array([x, y, z]) - clouds.append(pts) - return np.concatenate(clouds).astype(np.float32) - - -def points_in_box(points: np.ndarray, bbox_3d: List[float]) -> np.ndarray: - """Boolean mask of points inside an oriented 3D box.""" - x, y, z, l, w, h, yaw = bbox_3d - shifted = points - np.array([x, y, z]) - c, s = math.cos(-yaw), math.sin(-yaw) - rx = shifted[:, 0] * c - shifted[:, 1] * s - ry = shifted[:, 0] * s + shifted[:, 1] * c - rz = shifted[:, 2] - return (np.abs(rx) <= l / 2) & (np.abs(ry) <= w / 2) & (np.abs(rz) <= h / 2) - - -# ------------------------------------------------------------------ label generation - - -DEFAULT_ERROR_RATES = { - "FALSE_POSITIVE": 0.03, - "FALSE_NEGATIVE": 0.03, - "BAD_3D_BOX": 0.02, - "WRONG_ORIENTATION": 0.02, - "WRONG_POSITION": 0.02, - "LOW_CONFIDENCE": 0.02, - "SENSOR_DISAGREEMENT": 0.015, - "GRADER_DISAGREEMENT": 0.02, -} - - -def generate_labels( - store: EvalStore, - dataset: Dataset, - model_version: str = "model-v1", - error_rates: Optional[Dict[str, float]] = None, - degrade_classes: Optional[List[str]] = None, - seed_offset: int = 1, -) -> List[Annotation]: - """Create auto-generated labels (hypotheses) from GT with injected defects.""" - rates = dict(DEFAULT_ERROR_RATES) - if error_rates: - rates.update(error_rates) - degrade_classes = degrade_classes or [] - rng = np.random.default_rng(dataset.seed * 7919 + seed_offset) - - frames = sorted(store.where("frames", dataset_id=dataset.dataset_id), key=lambda f: f.index) - sparse = {(f, g) for f, g in dataset.generation_params.get("sparse_gt", [])} - - annotations: List[Annotation] = [] - injected_counts: Dict[str, int] = {e: 0 for e in ERROR_TYPES} - track_ann: Dict[str, List[str]] = {} - - for frame in frames: - for gt in frame.gt_boxes: - drop_rate = rates["FALSE_NEGATIVE"] + (0.30 if gt.class_name in degrade_classes else 0.0) - if rng.random() < drop_rate: - injected_counts["FALSE_NEGATIVE"] += 1 - continue - - x, y, z, l, w, h, yaw = gt.bbox_3d - # Nominal detector noise, proportional to object size (large boxes - # tolerate more absolute error at equal IoU). - sigma_pos = 0.012 * max(l, w) - x += float(rng.normal(0, sigma_pos)) - y += float(rng.normal(0, sigma_pos)) - z += float(rng.normal(0, 0.008 * h)) - l *= float(rng.normal(1, 0.012)) - w *= float(rng.normal(1, 0.012)) - h *= float(rng.normal(1, 0.012)) - yaw += float(rng.normal(0, math.radians(1.2))) - dist = math.hypot(x, y) - conf = float(np.clip(0.97 - dist / 200 + rng.normal(0, 0.04), 0.5, 0.99)) - - errors: List[str] = [] - if (frame.frame_id, gt.gt_id) in sparse: - errors.append("LOW_POINT_DENSITY") - r = rng.random() - if r < rates["WRONG_POSITION"]: - x += float(rng.choice([-1, 1])) * float(rng.uniform(1.5, 3.0)) - y += float(rng.choice([-1, 1])) * float(rng.uniform(1.0, 2.5)) - errors.append("WRONG_POSITION") - elif r < rates["WRONG_POSITION"] + rates["BAD_3D_BOX"]: - factor = float(rng.choice([0.45, 1.9, 2.3])) - l *= factor - w *= factor - errors.append("BAD_3D_BOX") - elif r < rates["WRONG_POSITION"] + rates["BAD_3D_BOX"] + rates["WRONG_ORIENTATION"]: - yaw += math.radians(float(rng.uniform(30, 90)) * float(rng.choice([-1, 1]))) - errors.append("WRONG_ORIENTATION") - if rng.random() < rates["LOW_CONFIDENCE"]: - conf = float(rng.uniform(0.12, 0.35)) - errors.append("LOW_CONFIDENCE") - if rng.random() < rates["SENSOR_DISAGREEMENT"]: - errors.append("SENSOR_DISAGREEMENT") - if rng.random() < rates["GRADER_DISAGREEMENT"]: - errors.append("GRADER_DISAGREEMENT") - for e in errors: - injected_counts[e] += 1 - - bbox_3d = [round(v, 4) for v in [x, y, z, l, w, h, yaw]] - ann = Annotation( - annotation_id=new_id("ann"), - dataset_id=dataset.dataset_id, - frame_id=frame.frame_id, - object_id=gt.track_instance_id, - class_name=gt.class_name, - confidence=round(conf, 4), - bbox_3d=bbox_3d, - bbox_2d=None if "SENSOR_DISAGREEMENT" in errors else project_bbox_2d(bbox_3d), - track_id=f"trk-{gt.track_instance_id}", - model="synthlab-detector", - model_version=model_version, - matched_gt_id=gt.gt_id, - injected_errors=errors, - ) - annotations.append(ann) - track_ann.setdefault(ann.track_id, []).append(ann.annotation_id) - - # False positives: labels with no supporting GT. - if rng.random() < rates["FALSE_POSITIVE"] * 6: - cls = CLASS_POOL[int(rng.integers(0, len(CLASS_POOL)))] - l, w, h = CLASS_DIMS[cls] - bx = float(rng.uniform(10, 60)) - by = float(rng.uniform(-15, 15)) - bbox_3d = [round(bx, 3), round(by, 3), round(h / 2, 3), l, w, h, round(float(rng.uniform(-math.pi, math.pi)), 3)] - ann = Annotation( - annotation_id=new_id("ann"), - dataset_id=dataset.dataset_id, - frame_id=frame.frame_id, - object_id=f"fp-{frame.frame_id}", - class_name=cls, - confidence=round(float(rng.uniform(0.4, 0.75)), 4), - bbox_3d=bbox_3d, - bbox_2d=project_bbox_2d(bbox_3d), - track_id=None, - model_version=model_version, - matched_gt_id=None, - injected_errors=["FALSE_POSITIVE"], - ) - annotations.append(ann) - injected_counts["FALSE_POSITIVE"] += 1 - - # Track-level errors: ID switches and fragmentation on long tracks. - long_tracks = [t for t, ids in track_ann.items() if len(ids) >= 10] - ann_by_id = {a.annotation_id: a for a in annotations} - if len(long_tracks) >= 2: - ta, tb = long_tracks[0], long_tracks[1] - for aid in track_ann[ta][len(track_ann[ta]) // 2:]: - ann_by_id[aid].track_id = tb - ann_by_id[aid].injected_errors.append("ID_SWITCH") - injected_counts["ID_SWITCH"] += 1 - if len(long_tracks) >= 3: - tc = long_tracks[2] - frag_id = f"{tc}-frag" - for aid in track_ann[tc][len(track_ann[tc]) // 2:]: - ann_by_id[aid].track_id = frag_id - ann_by_id[aid].injected_errors.append("TRACK_FRAGMENTATION") - injected_counts["TRACK_FRAGMENTATION"] += 1 - - for ann in annotations: - store.put("annotations", ann) - - # Track entities from final label track ids. - final_tracks: Dict[str, List[Annotation]] = {} - for a in annotations: - if a.track_id: - final_tracks.setdefault(a.track_id, []).append(a) - for tid, anns in final_tracks.items(): - store.put("tracks", Track( - track_id=tid, - dataset_id=dataset.dataset_id, - class_name=anns[0].class_name, - annotation_ids=[a.annotation_id for a in anns], - frame_ids=[a.frame_id for a in anns], - )) - - dataset.num_annotations = len(annotations) - dataset.generation_params["injected"] = {k: v for k, v in injected_counts.items() if v > 0} - matched = len([a for a in annotations if a.matched_gt_id]) - dataset.gt_coverage = round(matched / len(annotations), 4) if annotations else 0.0 - dataset.status = "labeled" - store.put("datasets", dataset) - store.audit("labels_generated", "Dataset", dataset.dataset_id, - f"{len(annotations)} labels by {model_version}; injected: {dataset.generation_params['injected']}") - return annotations - - -# ------------------------------------------------------------------ feature extraction - - -def annotation_features(store: EvalStore, dataset_id: str) -> Tuple[np.ndarray, List[str]]: - """Feature matrix for anomaly detection over all labels of a dataset.""" - anns = sorted(store.where("annotations", dataset_id=dataset_id), key=lambda a: a.annotation_id) - frames = {f.frame_id: f for f in store.where("frames", dataset_id=dataset_id)} - cache: Dict[str, np.ndarray] = {} - - feats, ids = [], [] - class_index = {c: i for i, c in enumerate(CLASS_DIMS)} - for a in anns: - if not a.bbox_3d: - continue - frame = frames.get(a.frame_id) - if frame is None: - continue - if a.frame_id not in cache: - cache[a.frame_id] = frame_points(store, frame) - pts = cache[a.frame_id] - x, y, z, l, w, h, yaw = a.bbox_3d - mask = points_in_box(pts, a.bbox_3d) - n_in = int(mask.sum()) - volume = max(l * w * h, 1e-3) - density = n_in / volume - exp_l, exp_w, exp_h = CLASS_DIMS.get(a.class_name, (2, 2, 2)) - dim_ratio = (l / exp_l + w / exp_w + h / exp_h) / 3 - onehot = [0.0] * len(class_index) - onehot[class_index.get(a.class_name, 0)] = 1.0 - feats.append([ - a.confidence, - math.hypot(x, y), - z, - l, w, h, - volume, - dim_ratio, - abs(math.sin(yaw)), - float(n_in), - density, - float(a.bbox_2d is None), - *onehot, - ]) - ids.append(a.annotation_id) - return (np.array(feats, dtype=np.float64) if feats else np.zeros((0, 12 + len(class_index)))), ids diff --git a/hf-space/sensorflow/evaluation/triage.py b/hf-space/sensorflow/evaluation/triage.py deleted file mode 100644 index 109838d1ffb46d48abd616eb1bb0b0d190cd173d..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/triage.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Strict, deterministic, configurable triage gate (spec §19-§21). - -Combines ML anomaly score, regression status, grader consensus, geometric -validation and track validation under a quality policy. Auto-release only when -ALL applicable gates pass. Every decision stores the policy id + values and -per-gate lines with actual vs threshold (explainability, spec §39). -""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Dict, List, Optional - -from pydantic import BaseModel, Field - -from sensorflow.evaluation.records import ( - Annotation, - AnomalyDetection, - CheckLine, - EvalStore, - GraderComparison, - TrackingEvidence, - TriageDecision, - ValidationResult, - new_id, -) - - -class QualityPolicy(BaseModel): - policy_id: str = "quality-policy-v1" - min_iou_3d: float = 0.80 - max_position_error_m: float = 0.50 - max_orientation_error_deg: float = 10.0 - min_points_in_box: int = 12 - min_consensus: float = 0.90 - max_anomaly_score: float = 0.90 - min_track_quality: float = 0.60 - min_confidence: float = 0.40 - require_sensor_consistency: bool = True - require_geometric_validation: bool = True - block_on_model_regression: bool = False - - def values(self) -> Dict[str, float]: - return {k: (float(v) if not isinstance(v, bool) else float(v)) - for k, v in self.model_dump().items() if k != "policy_id"} - - -POLICY_PATH = Path("runs/labeleval/quality_policy.json") - - -def load_policy(path: Path = POLICY_PATH) -> QualityPolicy: - if path.exists(): - try: - with open(path) as f: - return QualityPolicy.model_validate(json.load(f)) - except Exception: - pass - return QualityPolicy() - - -def save_policy(policy: QualityPolicy, path: Path = POLICY_PATH) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w") as f: - json.dump(policy.model_dump(), f, indent=2) - - -# Gate failure -> failure reason mapping (spec §21). -def triage_annotation( - store: EvalStore, - ann: Annotation, - validation: Optional[ValidationResult], - anomaly: Optional[AnomalyDetection], - grading: Optional[GraderComparison], - tracking: Optional[TrackingEvidence], - policy: QualityPolicy, - model_regressed: bool = False, -) -> TriageDecision: - lines: List[CheckLine] = [] - reasons: List[str] = [] - - def gate(name: str, actual, threshold, passed: bool, applicable: bool = True, reason: Optional[str] = None): - lines.append(CheckLine(gate=name, actual=actual, threshold=threshold, - passed=passed or not applicable, applicable=applicable)) - if applicable and not passed and reason and reason not in reasons: - reasons.append(reason) - - # --- geometric gates (only applicable when a reference exists) - has_ref = validation is not None and validation.iou_3d is not None - gate("iou_3d", validation.iou_3d if has_ref else "no reference", policy.min_iou_3d, - bool(has_ref and validation.iou_3d >= policy.min_iou_3d), applicable=has_ref, reason="LOW_IOU") - gate("position_error_m", validation.position_error if has_ref else "no reference", - policy.max_position_error_m, - bool(has_ref and validation.position_error <= policy.max_position_error_m), - applicable=has_ref, reason="POSITION_ERROR") - gate("orientation_error_deg", validation.orientation_error_deg if has_ref else "no reference", - policy.max_orientation_error_deg, - bool(has_ref and validation.orientation_error_deg <= policy.max_orientation_error_deg), - applicable=has_ref, reason="ORIENTATION_ERROR") - - # Unmatched labels (no reference GT at all) are inherently suspect: the - # independent evidence must carry them; flag as anomaly-checked FP risk. - if ann.matched_gt_id is None: - gate("has_reference_gt", False, True, False, applicable=True, reason="ANOMALY") - - # --- point support - n_pts = None - if validation is not None: - pts_line = next((c for c in validation.checks if c.gate == "points_in_box"), None) - n_pts = pts_line.actual if pts_line else None - gate("points_in_box", n_pts, policy.min_points_in_box, - bool(isinstance(n_pts, (int, float)) and n_pts >= policy.min_points_in_box), - applicable=n_pts is not None, reason="INSUFFICIENT_POINT_SUPPORT") - - # --- geometric validation verdict; map each failing check to its reason - CHECK_REASONS = { - "points_in_box": "INSUFFICIENT_POINT_SUPPORT", - "point_density_per_m3": "INSUFFICIENT_POINT_SUPPORT", - "box_occupancy": "INSUFFICIENT_POINT_SUPPORT", - "iou_3d_vs_reference": "LOW_IOU", - "position_error_m": "POSITION_ERROR", - "centroid_consistency_m": "POSITION_ERROR", - "ground_contact_error_m": "POSITION_ERROR", - "orientation_error_deg": "ORIENTATION_ERROR", - "box_dimensions_vs_class_prior": "LOW_IOU", - "dimension_error": "LOW_IOU", - "sensor_consistency_cam_lidar": "SENSOR_DISAGREEMENT", - } - if validation is not None and not validation.passed: - for c in validation.checks: - if c.applicable and not c.passed: - r = CHECK_REASONS.get(c.gate) - if r and r not in reasons: - reasons.append(r) - gate("geometric_validation", "passed" if (validation and validation.passed) else "failed", - "passed", bool(validation and validation.passed), - applicable=policy.require_geometric_validation, reason=None) - - # --- sensor consistency - gate("sensor_consistency", "consistent" if (validation is None or validation.sensor_consistent) else "disagreement", - "consistent", bool(validation is None or validation.sensor_consistent), - applicable=policy.require_sensor_consistency and validation is not None, reason="SENSOR_DISAGREEMENT") - - # --- anomaly - gate("anomaly_score", round(anomaly.score, 4) if anomaly else None, policy.max_anomaly_score, - bool(anomaly is None or not anomaly.is_anomaly), - applicable=anomaly is not None, reason="ANOMALY") - - # --- grader consensus - gate("grader_consensus", grading.consensus if grading else None, policy.min_consensus, - bool(grading is not None and grading.consensus is not None and grading.consensus >= policy.min_consensus), - applicable=grading is not None, reason="GRADER_DISAGREEMENT") - - # --- tracking - if tracking is not None: - gate("id_switch", tracking.id_switch, False, not tracking.id_switch, reason="ID_SWITCH") - gate("track_fragmentation", tracking.fragmentation, False, not tracking.fragmentation, - reason="TRACK_FRAGMENTATION") - gate("track_quality", tracking.track_quality, policy.min_track_quality, - bool(tracking.track_quality is None or tracking.track_quality >= policy.min_track_quality), - applicable=tracking.track_quality is not None, reason="TRACK_FRAGMENTATION") - - # --- confidence - gate("detection_confidence", ann.confidence, policy.min_confidence, - ann.confidence >= policy.min_confidence, reason="LOW_CONFIDENCE") - - # --- model regression - gate("model_regression", "regressed" if model_regressed else "stable", "stable", - not model_regressed, applicable=policy.block_on_model_regression, reason="MODEL_REGRESSION") - - all_pass = all(l.passed for l in lines if l.applicable) - status = "AUTO_GRADED" if all_pass else "FLAGGED" - - decision = TriageDecision( - decision_id=new_id("dec"), - annotation_id=ann.annotation_id, - status=status, - failure_reasons=reasons, - primary_failure_reason=reasons[0] if reasons else None, - gate_lines=lines, - policy_id=policy.policy_id, - policy_values=policy.values(), - ) - store.put("triage_decisions", decision) - return decision diff --git a/hf-space/sensorflow/evaluation/validation.py b/hf-space/sensorflow/evaluation/validation.py deleted file mode 100644 index d72e9a83847686a3bb9357692813eac37ced03f3..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/evaluation/validation.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Strict Quality Validation Engine: deterministic geometric plausibility checks. - -Compares LiDAR point cloud vs bounding-box proposal (spec §8) and computes -per-3D-box metrics with configurable tolerances (spec §9): 3D IoU, position -error, orientation error, dimension error, point density, point-in-box ratio, -ground contact error, plus sensor-consistency checks. -""" - -from __future__ import annotations - -import math -from typing import Dict, List, Optional - -import numpy as np - -from sensorflow.evaluation.records import ( - Annotation, - CheckLine, - EvalStore, - Frame, - ValidationResult, -) -from sensorflow.evaluation.synthetic import ( - CLASS_DIMS, - frame_points, - points_in_box, - project_bbox_2d, -) -from sensorflow.metrics.perception_3d import bev_iou - -DEFAULT_TOLERANCES: Dict[str, float] = { - "min_points_in_box": 12, - "min_point_density": 0.4, # points per m^3 - "min_box_occupancy": 0.05, # fraction of nearby points captured - "max_dim_ratio": 1.6, # vs class prior - "min_dim_ratio": 0.55, - "max_centroid_offset_m": 1.2, # points centroid vs box center (BEV) - "max_ground_contact_error_m": 0.5, - "max_orientation_track_dev_deg": 45.0, - "min_iou_3d": 0.5, # plausibility floor (triage gate is stricter) - "max_position_error_m": 1.0, - "max_orientation_error_deg": 30.0, - "max_dimension_error": 0.5, -} - - -def iou_3d(box_a: List[float], box_b: List[float]) -> float: - """BEV IoU extended with vertical overlap.""" - biou = bev_iou(box_a, box_b) - za0, za1 = box_a[2] - box_a[5] / 2, box_a[2] + box_a[5] / 2 - zb0, zb1 = box_b[2] - box_b[5] / 2, box_b[2] + box_b[5] / 2 - inter_h = max(0.0, min(za1, zb1) - max(za0, zb0)) - union_h = max(za1, zb1) - min(za0, zb0) - if union_h <= 0: - return 0.0 - return biou * (inter_h / union_h) * (union_h / max(box_a[5], box_b[5])) # approx volumetric - - -def validate_annotation( - store: EvalStore, - ann: Annotation, - frame: Frame, - points: Optional[np.ndarray] = None, - tolerances: Optional[Dict[str, float]] = None, -) -> ValidationResult: - tol = dict(DEFAULT_TOLERANCES) - if tolerances: - tol.update(tolerances) - - checks: List[CheckLine] = [] - result = ValidationResult(annotation_id=ann.annotation_id) - - if not ann.bbox_3d: - result.passed = False - result.checks = [CheckLine(gate="has_bbox_3d", actual=False, threshold=True, passed=False)] - store.put("validations", result) - return result - - if points is None: - points = frame_points(store, frame) - - x, y, z, l, w, h, yaw = ann.bbox_3d - volume = max(l * w * h, 1e-3) - - # --- point support - mask = points_in_box(points, ann.bbox_3d) - n_in = int(mask.sum()) - density = n_in / volume - result.point_density = round(density, 3) - - checks.append(CheckLine(gate="points_in_box", actual=n_in, - threshold=tol["min_points_in_box"], - passed=n_in >= tol["min_points_in_box"])) - checks.append(CheckLine(gate="point_density_per_m3", actual=round(density, 2), - threshold=tol["min_point_density"], - passed=density >= tol["min_point_density"])) - - # --- box occupancy: fraction of points near the box that fall inside it - near = points[(np.abs(points[:, 0] - x) < max(l, 4)) & (np.abs(points[:, 1] - y) < max(w, 4))] - above_ground = near[near[:, 2] > 0.15] if len(near) else near - occupancy = n_in / max(len(above_ground), 1) - result.point_in_box_ratio = round(min(occupancy, 1.0), 4) - checks.append(CheckLine(gate="box_occupancy", actual=round(occupancy, 3), - threshold=tol["min_box_occupancy"], - passed=occupancy >= tol["min_box_occupancy"])) - - # --- dimension plausibility vs class prior - exp = CLASS_DIMS.get(ann.class_name) - if exp: - ratio = float(np.mean([l / exp[0], w / exp[1], h / exp[2]])) - ok = tol["min_dim_ratio"] <= ratio <= tol["max_dim_ratio"] - checks.append(CheckLine(gate="box_dimensions_vs_class_prior", actual=round(ratio, 3), - threshold=f"[{tol['min_dim_ratio']}, {tol['max_dim_ratio']}]", passed=ok)) - else: - checks.append(CheckLine(gate="box_dimensions_vs_class_prior", actual="unknown_class", - threshold="known class", passed=True, applicable=False)) - - # --- centroid consistency: LiDAR centroid vs box center - if n_in >= 3: - centroid = points[mask].mean(axis=0) - offset = math.hypot(centroid[0] - x, centroid[1] - y) - checks.append(CheckLine(gate="centroid_consistency_m", actual=round(offset, 3), - threshold=tol["max_centroid_offset_m"], - passed=offset <= tol["max_centroid_offset_m"])) - else: - checks.append(CheckLine(gate="centroid_consistency_m", actual="insufficient points", - threshold=tol["max_centroid_offset_m"], passed=False, applicable=n_in > 0)) - - # --- object-to-ground relationship - ground_err = abs((z - h / 2) - 0.0) - result.ground_contact_error = round(ground_err, 3) - checks.append(CheckLine(gate="ground_contact_error_m", actual=round(ground_err, 3), - threshold=tol["max_ground_contact_error_m"], - passed=ground_err <= tol["max_ground_contact_error_m"])) - - # --- sensor consistency: LiDAR box should have a camera projection when - # visible; a projectable box missing a 2D detection is a disagreement. - expected_2d = project_bbox_2d(ann.bbox_3d) - sensor_ok = not (expected_2d is not None and ann.bbox_2d is None) - result.sensor_consistent = sensor_ok - checks.append(CheckLine(gate="sensor_consistency_cam_lidar", - actual="2d_present" if ann.bbox_2d else "2d_missing", - threshold="camera+lidar agree", passed=sensor_ok, - applicable=expected_2d is not None)) - - # --- reference comparison (only when a reference ground truth exists) - gt = next((g for g in frame.gt_boxes if g.gt_id == ann.matched_gt_id), None) - if gt is not None: - iou = iou_3d(ann.bbox_3d, gt.bbox_3d) - pos_err = math.hypot(x - gt.bbox_3d[0], y - gt.bbox_3d[1]) - yaw_err = abs(yaw - gt.bbox_3d[6]) % (2 * math.pi) - yaw_err = min(yaw_err, 2 * math.pi - yaw_err) - yaw_err_deg = math.degrees(yaw_err) - dim_err = float(np.mean([abs(ann.bbox_3d[i] - gt.bbox_3d[i]) / max(gt.bbox_3d[i], 0.1) for i in (3, 4, 5)])) - - result.iou_3d = round(iou, 4) - result.position_error = round(pos_err, 3) - result.orientation_error_deg = round(yaw_err_deg, 2) - result.dimension_error = round(dim_err, 4) - - checks.append(CheckLine(gate="iou_3d_vs_reference", actual=round(iou, 3), - threshold=tol["min_iou_3d"], passed=iou >= tol["min_iou_3d"])) - checks.append(CheckLine(gate="position_error_m", actual=round(pos_err, 3), - threshold=tol["max_position_error_m"], - passed=pos_err <= tol["max_position_error_m"])) - checks.append(CheckLine(gate="orientation_error_deg", actual=round(yaw_err_deg, 1), - threshold=tol["max_orientation_error_deg"], - passed=yaw_err_deg <= tol["max_orientation_error_deg"])) - checks.append(CheckLine(gate="dimension_error", actual=round(dim_err, 3), - threshold=tol["max_dimension_error"], - passed=dim_err <= tol["max_dimension_error"])) - else: - checks.append(CheckLine(gate="iou_3d_vs_reference", actual="no reference GT", - threshold=tol["min_iou_3d"], passed=True, applicable=False)) - - result.checks = checks - result.passed = all(c.passed for c in checks if c.applicable) - store.put("validations", result) - return result diff --git a/hf-space/sensorflow/launch_gate_evaluator.py b/hf-space/sensorflow/launch_gate_evaluator.py deleted file mode 100644 index 861c58a33ed52c1e07387d8de3eac82ba6216060..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/launch_gate_evaluator.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Launch gate evaluator: block export until quality thresholds pass.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, Dict, List, Optional - - -DEFAULT_THRESHOLDS_PATH = Path("runs/pipeline/gate_thresholds.json") - -DEFAULT_THRESHOLDS = { - "map_3d": 0.65, - "orientation_error_deg": 5.0, - "id_swap_rate": 0.02, - "track_fragmentation_rate": 0.05, - "position_error_m": 2.0, -} - - -class LaunchGateEvaluator: - """Validate benchmark metrics against safety thresholds before export.""" - - def __init__(self, thresholds_path: Optional[Path] = None): - self.thresholds_path = thresholds_path or DEFAULT_THRESHOLDS_PATH - self.thresholds = self._load_thresholds() - - def _load_thresholds(self) -> Dict[str, float]: - if self.thresholds_path.exists(): - with open(self.thresholds_path) as f: - return json.load(f) - self.thresholds_path.parent.mkdir(parents=True, exist_ok=True) - with open(self.thresholds_path, "w") as f: - json.dump(DEFAULT_THRESHOLDS, f, indent=2) - return DEFAULT_THRESHOLDS.copy() - - def evaluate(self, sequence_id: str) -> Dict[str, Any]: - metric_path = Path("runs/pipeline") / sequence_id / "benchmark" / "metric_card.json" - if not metric_path.exists(): - return { - "passed": False, - "failures": [{"metric": "benchmark", "message": "No benchmark results found. Run quality gate first."}], - "blocked_stages": ["export"], - } - - with open(metric_path) as f: - metrics = json.load(f) - - failures: List[Dict] = [] - checks = [ - ("map_3d", metrics.get("map_3d", 0), self.thresholds["map_3d"], ">="), - ("orientation_error_deg", metrics.get("orientation_error_deg", 999), self.thresholds["orientation_error_deg"], "<="), - ("id_swap_rate", metrics.get("id_swap_rate", 1), self.thresholds["id_swap_rate"], "<="), - ("track_fragmentation_rate", metrics.get("track_fragmentation_rate", 1), self.thresholds["track_fragmentation_rate"], "<="), - ] - for name, value, threshold, op in checks: - failed = (value < threshold) if op == ">=" else (value > threshold) - if failed: - failures.append({ - "metric": name, - "value": value, - "threshold": threshold, - "operator": op, - }) - - passed = len(failures) == 0 - result = { - "passed": passed, - "failures": failures, - "blocked_stages": [] if passed else ["export"], - "metrics": metrics, - } - - gate_path = Path("runs/pipeline") / sequence_id / "launch_gate.json" - gate_path.parent.mkdir(parents=True, exist_ok=True) - with open(gate_path, "w") as f: - json.dump(result, f, indent=2) - - state_path = Path("runs/pipeline/state.json") - state = {} - if state_path.exists(): - with open(state_path) as f: - state = json.load(f) - state.setdefault(sequence_id, {})["launch_gate_passed"] = passed - with open(state_path, "w") as f: - json.dump(state, f, indent=2) - - return result - - def is_export_allowed(self, sequence_id: str) -> bool: - gate_path = Path("runs/pipeline") / sequence_id / "launch_gate.json" - if gate_path.exists(): - with open(gate_path) as f: - return json.load(f).get("passed", False) - return False diff --git a/hf-space/sensorflow/megaeval/__init__.py b/hf-space/sensorflow/megaeval/__init__.py deleted file mode 100644 index d6f4a1f210fe828c2f9e68e261aa3e6012d41ec9..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/megaeval/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -"""megaeval — aggregate-first, risk-first evaluation layer for mega-scale populations. - -Mental model (maps 1:1 to modules): - - Dataset (immutable population) -> population.py (partitioned npz storage) - Evaluation Run (first-class job) -> runs.py (async workers -> partial stats -> reduce) - Population / Cohort -> cube.py (metric cube + query router + cache) - Container -> runs.py (container-level aggregate table) - Annotation (forensic drill-down) -> runs.py (per-object partitions, loaded on demand) - -Single-node stand-ins for distributed infra (see ARCHITECTURE.md): - npz partitions ~ Parquet files in an Iceberg table - worker thread pool ~ Spark executors / Flink task slots - partial-stats+reduce~ map-side combine + shuffle reduce - QueryRouter ~ OLAP engine (Druid/Pinot) in front of the lakehouse -""" diff --git a/hf-space/sensorflow/megaeval/analysis.py b/hf-space/sensorflow/megaeval/analysis.py deleted file mode 100644 index eca9bcadc6b5418d331d557a0a90dd61aa48c82b..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/megaeval/analysis.py +++ /dev/null @@ -1,314 +0,0 @@ -"""Cross-run analysis: model-vs-model comparison, distribution shift, the "why" -layer, and hybrid similarity search. - -All computations run on cubes and the error index — never on raw records. -""" - -from __future__ import annotations - -import json -import os -from typing import Dict, List, Optional - -import numpy as np -import pandas as pd - -from sensorflow.megaeval import cube as cube_mod -from sensorflow.megaeval import population as pop_mod -from sensorflow.megaeval.population import DIMENSIONS, DIM_NAMES -from sensorflow.megaeval.runs import ERROR_TYPES, EvaluationRun, MegaStore, run_dir - -# ------------------------------------------------------------------ model vs model - -DEFAULT_PROMOTION_POLICY = { - "max_recall_drop": 0.010, - "max_precision_drop": 0.015, - "max_safety_recall_drop": 0.005, - "max_cohort_recall_drop": 0.050, - "min_cohort_support": 200, - "cohort_dims": ["road_type", "lighting", "class"], -} - -_HEADLINE_METRICS = ["precision", "recall", "f1", "mean_iou", "safety_recall", "anomaly_rate"] - - -def compare_runs(store: MegaStore, candidate: EvaluationRun, baseline: EvaluationRun, - policy: Optional[Dict] = None) -> Dict: - policy = {**DEFAULT_PROMOTION_POLICY, **(policy or {})} - cand_art, base_art = store.artifacts(candidate.run_id), store.artifacts(baseline.run_id) - cand_cube, base_cube = cand_art["cube"], base_art["cube"] - - headline = [] - for m in _HEADLINE_METRICS: - c, b = candidate.headline.get(m), baseline.headline.get(m) - if c is None or b is None: - continue - headline.append({"metric": m, "baseline": round(b, 4), "candidate": round(c, 4), - "delta": round(c - b, 4)}) - - per_class = [] - for cls in DIMENSIONS["class"]: - c, b = candidate.per_class.get(cls), baseline.per_class.get(cls) - if not c or not b: - continue - per_class.append({ - "class": cls, "n": c.get("n"), - "recall_baseline": b.get("recall"), "recall_candidate": c.get("recall"), - "recall_delta": round((c.get("recall") or 0) - (b.get("recall") or 0), 4), - "precision_baseline": b.get("precision"), "precision_candidate": c.get("precision"), - "precision_delta": round((c.get("precision") or 0) - (b.get("precision") or 0), 4), - }) - - dims = policy["cohort_dims"] - cand_rows, _ = cube_mod.aggregate(cand_cube, None, dims, ["n", "recall", "precision"], 2000) - base_rows, _ = cube_mod.aggregate(base_cube, None, dims, ["n", "recall", "precision"], 2000) - base_map = {tuple(r[d] for d in dims): r for r in base_rows} - cohorts = [] - for r in cand_rows: - key = tuple(r[d] for d in dims) - b = base_map.get(key) - if not b or r["n"] < policy["min_cohort_support"]: - continue - if r.get("recall") is None or b.get("recall") is None: - continue - cohorts.append({ - "cohort": "/".join(key), **{d: r[d] for d in dims}, "n": r["n"], - "recall_baseline": b["recall"], "recall_candidate": r["recall"], - "recall_delta": round(r["recall"] - b["recall"], 4), - }) - cohorts.sort(key=lambda x: x["recall_delta"]) - regressions = [c for c in cohorts if c["recall_delta"] < -policy["max_cohort_recall_drop"]] - - blockers: List[str] = [] - hd = {h["metric"]: h["delta"] for h in headline} - if hd.get("recall", 0) < -policy["max_recall_drop"]: - blockers.append(f"headline recall dropped {hd['recall']:+.4f} " - f"(policy allows -{policy['max_recall_drop']})") - if hd.get("precision", 0) < -policy["max_precision_drop"]: - blockers.append(f"headline precision dropped {hd['precision']:+.4f}") - if hd.get("safety_recall", 0) < -policy["max_safety_recall_drop"]: - blockers.append(f"safety recall dropped {hd['safety_recall']:+.4f} " - f"(policy allows -{policy['max_safety_recall_drop']})") - for c in regressions[:5]: - blockers.append( - f"REGRESSION: {c['cohort']} recall " - f"{c['recall_baseline']*100:.1f}% -> {c['recall_candidate']*100:.1f}% " - f"(Δ {c['recall_delta']*100:+.1f})") - - # integrate with the existing regression tracker so the Regression page shows it - try: - from sensorflow.evaluation import regression as legacy_regression - from sensorflow.evaluation.records import get_store as get_legacy_store - legacy_regression.compare_runs( - get_legacy_store(), - current_metrics={"precision": candidate.headline.get("precision"), - "recall": candidate.headline.get("recall"), - "safety_critical_recall": candidate.headline.get("safety_recall"), - "mean_iou_3d": candidate.headline.get("mean_iou")}, - model_version=candidate.model_version, - dataset_version=candidate.population_id, - run_id=candidate.run_id, - baseline_metrics={"precision": baseline.headline.get("precision"), - "recall": baseline.headline.get("recall"), - "safety_critical_recall": baseline.headline.get("safety_recall"), - "mean_iou_3d": baseline.headline.get("mean_iou")}, - baseline_version=baseline.model_version, - affected_classes=[c["class"] for c in per_class if c["recall_delta"] < -0.03], - affected_scenarios=sorted({c.get("road_type", "") for c in regressions[:5] if c}), - ) - except Exception: - pass - - return { - "candidate": {"run_id": candidate.run_id, "model_version": candidate.model_version}, - "baseline": {"run_id": baseline.run_id, "model_version": baseline.model_version}, - "headline_deltas": headline, - "per_class": per_class, - "worst_cohorts": cohorts[:15], - "regressions": regressions[:10], - "policy": policy, - "recommendation": "DO_NOT_PROMOTE" if blockers else "PROMOTE", - "blockers": blockers, - } - - -# ------------------------------------------------------------------ distribution shift - - -def distribution_shift(store: MegaStore, run: EvaluationRun, - min_eval_count: int = 300, rel_threshold: float = 0.35) -> Dict: - cache_path = os.path.join(run_dir(run.run_id), "shift.json") - if os.path.exists(cache_path): - with open(cache_path) as f: - return json.load(f) - - meta = pop_mod.load_meta(run.population_id) - cube_df = store.artifacts(run.run_id)["cube"] - overall_rows, _ = cube_mod.aggregate(cube_df, None, None, ["recall"], 1) - overall_recall = overall_rows[0].get("recall") - - shifts = [] - for row in meta.get("train_mix", []): - train, evals = row["train_share"], row["eval_share"] - if train <= 0 or row["eval_count"] < min_eval_count: - continue - rel = (evals - train) / train - if abs(rel) < rel_threshold: - continue - filt = {"class": [row["class"]], "weather": [row["weather"]], - "lighting": [row["lighting"]]} - crows, _ = cube_mod.aggregate(cube_df, filt, None, ["n", "recall", "precision"], 1) - c = crows[0] if crows else {} - shifts.append({ - "cohort": f"{row['class']}/{row['weather']}/{row['lighting']}", - "class": row["class"], "weather": row["weather"], "lighting": row["lighting"], - "train_share": round(train, 5), "eval_share": round(evals, 5), - "relative_change": round(rel, 4), - "eval_count": row["eval_count"], - "cohort_recall": c.get("recall"), - "overall_recall": overall_recall, - "recall_gap": (round(c["recall"] - overall_recall, 4) - if c.get("recall") is not None and overall_recall is not None else None), - }) - shifts.sort(key=lambda s: abs(s["relative_change"]) * (1 + abs(s.get("recall_gap") or 0) * 5), - reverse=True) - - # surface the worst shifts as alerts in the existing alerting system (once per run) - try: - from sensorflow.evaluation.records import Alert, get_store as get_legacy_store, new_id - legacy = get_legacy_store() - for s in shifts[:3]: - if s.get("recall_gap") is not None and s["recall_gap"] < -0.03: - legacy.put("alerts", Alert( - alert_id=new_id("alert"), kind="distribution_shift", - severity="critical" if s["recall_gap"] < -0.08 else "warning", - message=(f"Distribution shift {s['cohort']}: train {s['train_share']*100:.1f}% " - f"-> eval {s['eval_share']*100:.1f}% " - f"(rel {s['relative_change']*100:+.0f}%), cohort recall " - f"{(s['cohort_recall'] or 0)*100:.1f}% vs overall " - f"{(s['overall_recall'] or 0)*100:.1f}%"), - evidence_page="command", evidence_id=run.run_id)) - legacy.save() - except Exception: - pass - - result = {"run_id": run.run_id, "shifts": shifts[:20], - "method": "train mix vs eval mix over (class, weather, lighting); " - "relative share change with cube recall impact", - "thresholds": {"min_eval_count": min_eval_count, "rel_threshold": rel_threshold}} - with open(cache_path, "w") as f: - json.dump(result, f) - return result - - -# ------------------------------------------------------------------ the "why" layer - - -_FACTOR_ORDER = ["occlusion", "low_illumination", "long_range", "sensor_disagreement", "other"] - - -def why(store: MegaStore, run: EvaluationRun, filters: Optional[Dict] = None, - metric: str = "recall") -> Dict: - """Decompose the failure population of a filtered slice by primary factor.""" - art = store.artifacts(run.run_id) - errors = art.get("errors") - cube_df = art["cube"] - if errors is None or not len(errors): - return {"factors": [], "top_cohorts": [], "failure_count": 0} - - mask = np.ones(len(errors), dtype=bool) - fail_types = [ERROR_TYPES.index(t) for t in ("FN", "FP", "LOCALIZATION")] - if metric == "recall": - fail_types = [ERROR_TYPES.index("FN"), ERROR_TYPES.index("LOCALIZATION")] - elif metric == "precision": - fail_types = [ERROR_TYPES.index("FP"), ERROR_TYPES.index("LOCALIZATION")] - mask &= errors["error_type"].isin(fail_types).to_numpy() - for dim, values in (filters or {}).items(): - if dim not in DIM_NAMES: - continue - vocab = DIMENSIONS[dim] - codes = [vocab.index(v) for v in (values if isinstance(values, list) else [values]) - if v in vocab] - mask &= errors[dim].isin(codes).to_numpy() - sub = errors[mask] - total = len(sub) - if total == 0: - return {"factors": [], "top_cohorts": [], "failure_count": 0} - - occl = sub["occlusion"].to_numpy() >= 1 - night = sub["lighting"].to_numpy() == 1 - far = sub["distance_band"].to_numpy() >= 2 - sensor = sub["sensor_disagree"].to_numpy().astype(bool) - factor = np.full(total, 4, dtype=np.int8) # other - factor[sensor] = 3 - factor[far] = 2 - factor[night] = 1 - factor[occl] = 0 # priority: occlusion > low light > range > sensor > other - counts = np.bincount(factor, minlength=5) - factors = [{"factor": _FACTOR_ORDER[i], "count": int(counts[i]), - "share": round(float(counts[i]) / total, 4)} - for i in range(5) if counts[i] > 0] - factors.sort(key=lambda x: -x["count"]) - - rows, _ = cube_mod.aggregate(cube_df, filters, ["class", "lighting", "weather"], - ["n", metric, "fn", "fp"], 2000) - rows = [r for r in rows if r["n"] >= 150 and r.get(metric) is not None] - rows.sort(key=lambda r: r[metric]) - top = [{"cohort": f"{r['class']}/{r['lighting']}/{r['weather']}", - "n": r["n"], metric: r[metric], "fn": r.get("fn"), "fp": r.get("fp")} - for r in rows[:8]] - - return {"metric": metric, "filters": filters or {}, "failure_count": int(total), - "factors": factors, "top_cohorts": top, - "method": "primary-factor attribution over the error index " - "(priority: occlusion > low illumination > long range > " - "sensor disagreement > other)"} - - -# ------------------------------------------------------------------ hybrid similarity - - -def similar_containers(store: MegaStore, run: EvaluationRun, container_id: int, - filters: Optional[Dict] = None, k: int = 12) -> Dict: - art = store.artifacts(run.run_id) - ids, emb = art["emb_ids"], art["emb"] - containers = art["containers"].set_index("container_id") - pos = np.where(ids == container_id)[0] - if pos.size == 0: - raise KeyError(f"Unknown container {container_id}") - q = emb[pos[0]] - sims = emb @ q - - mask = np.ones(ids.shape[0], dtype=bool) - for dim, values in (filters or {}).items(): - if dim not in pop_mod.CONTAINER_DIMS: - continue - vocab = DIMENSIONS[dim] - codes = [vocab.index(v) for v in (values if isinstance(values, list) else [values]) - if v in vocab] - col = containers.loc[ids, dim].to_numpy() - mask &= np.isin(col, codes) - mask[pos[0]] = False - sims = np.where(mask, sims, -np.inf) - - kk = min(k, int(mask.sum())) - if kk <= 0: - return {"query_container": container_id, "results": []} - top = np.argpartition(-sims, kk - 1)[:kk] - top = top[np.argsort(-sims[top])] - results = [] - for i in top.tolist(): - cid = int(ids[i]) - row = containers.loc[cid] - results.append({ - "container_id": cid, - "similarity": round(float(sims[i]), 4), - **{dim: DIMENSIONS[dim][int(row[dim])] for dim in pop_mod.CONTAINER_DIMS}, - "n_objects": int(row["n_objects"]), - "fn": int(row["fn"]), "fp": int(row["fp"]), - "anomalies": int(row["anomalies"]), - "risk_score": round(float(row["risk_score"]), 4), - }) - return {"query_container": container_id, - "retrieval": "cosine over 32-d structural embeddings + structured filters", - "results": results} diff --git a/hf-space/sensorflow/megaeval/api.py b/hf-space/sensorflow/megaeval/api.py deleted file mode 100644 index 4dc5ad0b7e0f20b78b3ff4ddce4a95944765acc2..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/megaeval/api.py +++ /dev/null @@ -1,387 +0,0 @@ -"""FastAPI router for the aggregate-first mega-scale evaluation layer.""" - -from __future__ import annotations - -from typing import Dict, List, Optional - -import numpy as np -from fastapi import APIRouter, HTTPException -from pydantic import BaseModel, Field - -from sensorflow.megaeval import analysis, errors as errors_mod, population as pop_mod -from sensorflow.megaeval import sampling -from sensorflow.megaeval.cube import SKETCH_METRICS -from sensorflow.megaeval.population import DIMENSIONS -from sensorflow.megaeval.runs import CONTAINER_STATUS, EvaluationRun, get_mega_store - -router = APIRouter(prefix="/api") - - -def _run_or_404(run_id: str) -> EvaluationRun: - run = get_mega_store().runs.get(run_id) - if run is None: - raise HTTPException(404, f"Unknown evaluation run {run_id}") - return run - - -def _published_or_409(run_id: str) -> EvaluationRun: - run = _run_or_404(run_id) - if run.status != "published": - raise HTTPException(409, f"Run {run_id} is {run.status}; results are available " - "once it is published") - return run - - -# ------------------------------------------------------------------ requests - - -class GeneratePopulationRequest(BaseModel): - name: str = "mega-perception" - num_objects: int = Field(320_000, ge=1_000, le=1_200_000) - seed: int = 42 - - -class CreateRunRequest(BaseModel): - population_id: str - model_version: str = "model-v42" - overrides: Dict = Field(default_factory=dict) - worker_delay_s: float = Field(0.5, ge=0.0, le=5.0) - workers: int = Field(4, ge=1, le=8) - seed: Optional[int] = None - label_version: str = "labels-v1" - - -class EvaluationQueryRequest(BaseModel): - evaluation_id: str - filters: Dict[str, List[str]] = Field(default_factory=dict) - metrics: List[str] = Field(default_factory=list) - group_by: List[str] = Field(default_factory=list) - limit: int = Field(200, ge=1, le=2000) - - -class ErrorSearchRequest(BaseModel): - run_id: str - error_types: List[str] = Field(default_factory=list) - filters: Dict[str, List[str]] = Field(default_factory=dict) - confidence_max: Optional[float] = None - confidence_min: Optional[float] = None - risk_min: Optional[float] = None - severity_min: Optional[float] = None - safety_only: bool = False - limit_containers: int = Field(25, ge=1, le=100) - - -class CompareRequest(BaseModel): - candidate_run_id: str - baseline_run_id: str - policy: Dict = Field(default_factory=dict) - - -class WhyRequest(BaseModel): - run_id: str - filters: Dict[str, List[str]] = Field(default_factory=dict) - metric: str = "recall" - - -class SimilarityRequest(BaseModel): - run_id: str - container_id: int - filters: Dict[str, List[str]] = Field(default_factory=dict) - k: int = Field(12, ge=1, le=50) - - -class ReviewPlanRequest(BaseModel): - target_n: Optional[int] = Field(None, ge=100, le=20_000) - - -# ------------------------------------------------------------------ populations - - -@router.post("/megaeval/population/generate") -def generate_population(req: GeneratePopulationRequest): - meta = pop_mod.generate_population(name=req.name, num_objects=req.num_objects, - seed=req.seed) - return meta - - -@router.get("/megaeval/populations") -def list_populations(): - return {"populations": pop_mod.list_populations()} - - -@router.get("/megaeval/dimensions") -def dimensions(): - return {"dimensions": DIMENSIONS} - - -# ------------------------------------------------------------------ runs (async jobs) - - -@router.post("/megaeval/runs") -def create_run(req: CreateRunRequest): - store = get_mega_store() - try: - run = store.create_run(population_id=req.population_id, - model_version=req.model_version, - overrides=req.overrides, - seed=req.seed, - worker_delay_s=req.worker_delay_s, - workers=req.workers, - label_version=req.label_version) - except KeyError as e: - raise HTTPException(404, str(e)) - store.start_async(run) - return run.to_dict() - - -@router.get("/megaeval/runs") -def list_runs(): - store = get_mega_store() - runs = sorted(store.runs.values(), key=lambda r: r.created_at, reverse=True) - return {"runs": [r.to_dict() for r in runs]} - - -@router.get("/megaeval/runs/{run_id}") -def get_run(run_id: str): - return _run_or_404(run_id).to_dict() - - -@router.get("/megaeval/status") -def megaeval_status(): - store = get_mega_store() - active = [r.progress_dict() for r in store.runs.values() - if r.status in ("queued", "running", "reducing", "materializing")] - return {"active_runs": active, "cache": store.router.cache.stats(), - "total_runs": len(store.runs)} - - -# ------------------------------------------------------------------ THE query API - - -@router.post("/evaluations/query") -def evaluations_query(req: EvaluationQueryRequest): - """Aggregate-first query endpoint. Routed: cache -> cube -> record scan.""" - store = get_mega_store() - run = _published_or_409(req.evaluation_id) - art = store.artifacts(run.run_id) - - def scan_fn(filters, group_by, metrics, limit): - return store.scan_records(run, filters, group_by, metrics, limit) - - return store.router.query(run, art["cube"], art.get("sketches", {}), - req.filters, req.metrics, req.group_by, - req.limit, scan_fn=scan_fn) - - -@router.get("/megaeval/cache") -def cache_stats(): - return get_mega_store().router.cache.stats() - - -# ------------------------------------------------------------------ quality funnel - - -@router.get("/megaeval/runs/{run_id}/funnel") -def quality_funnel(run_id: str): - store = get_mega_store() - run = _published_or_409(run_id) - art = store.artifacts(run_id) - errs = art.get("errors") - review = art.get("review") or {} - n_err = int(len(errs)) if errs is not None else 0 - n_high = int((errs["risk_score"] >= 0.6).sum()) if errs is not None and len(errs) else 0 - results = (review.get("results") or {}) - prec = results.get("precision") or {} - stages = [ - {"stage": "Objects Evaluated", "count": int(run.objects_total)}, - {"stage": "Containers", "count": int(run.headline.get("containers", 0))}, - {"stage": "Potential Errors", "count": n_err}, - {"stage": "High-Confidence Errors", "count": n_high}, - {"stage": "Review Sample", "count": int((review.get("funnel") or {}).get("statistically_selected", 0))}, - {"stage": "Human Verified", "count": int((review.get("funnel") or {}).get("reviewed", 0))}, - ] - base = stages[0]["count"] or 1 - for s in stages: - s["pct_of_population"] = round(s["count"] / base * 100, 3) - return {"stages": stages, - "estimated_precision": ({"estimate": prec.get("estimate"), - "ci_low": prec.get("ci_low"), - "ci_high": prec.get("ci_high"), - "n_reviewed": prec.get("n_reviewed")} - if prec else None)} - - -# ------------------------------------------------------------------ containers - - -_SORT_PRESETS = { - "worst_recall": ("recall", True), - "worst_precision": ("precision", True), - "worst_iou": ("mean_iou", True), - "most_anomalies": ("anomalies", False), - "least_verified": ("reviewed", True), - "highest_risk": ("risk_score", False), -} - - -@router.get("/megaeval/runs/{run_id}/containers") -def containers(run_id: str, sort: str = "highest_risk", limit: int = 50, offset: int = 0): - store = get_mega_store() - _published_or_409(run_id) - art = store.artifacts(run_id) - df = art["containers"].copy() - tp, fp, fn = df["tp"], df["fp"], df["fn"] - with np.errstate(divide="ignore", invalid="ignore"): - df["recall"] = np.where((tp + fn) > 0, tp / (tp + fn), np.nan) - df["precision"] = np.where((tp + fp) > 0, tp / (tp + fp), np.nan) - df["mean_iou"] = np.where(tp > 0, df["sum_iou"] / tp, np.nan) - metric, ascending = _SORT_PRESETS.get(sort, _SORT_PRESETS["highest_risk"]) - df = df.sort_values(metric, ascending=ascending, na_position="last") - total = len(df) - page = df.iloc[offset:offset + max(1, min(limit, 500))] - rows = [] - for _, r in page.iterrows(): - rows.append({ - "container_id": int(r["container_id"]), - **{dim: DIMENSIONS[dim][int(r[dim])] for dim in pop_mod.CONTAINER_DIMS}, - "n_objects": int(r["n_objects"]), - "tp": int(r["tp"]), "fp": int(r["fp"]), "fn": int(r["fn"]), - "anomalies": int(r["anomalies"]), - "reviewed": int(r["reviewed"]), "verified": int(r["verified"]), - "recall": None if np.isnan(r["recall"]) else round(float(r["recall"]), 4), - "precision": None if np.isnan(r["precision"]) else round(float(r["precision"]), 4), - "mean_iou": None if np.isnan(r["mean_iou"]) else round(float(r["mean_iou"]), 4), - "risk_score": round(float(r["risk_score"]), 4), - "status": CONTAINER_STATUS[int(r["status"])], - }) - return {"total": total, "sort": sort, "rows": rows} - - -@router.get("/megaeval/runs/{run_id}/containers/{container_id}/objects") -def container_objects(run_id: str, container_id: int): - store = get_mega_store() - run = _published_or_409(run_id) - try: - rows = store.container_objects(run, container_id) - except FileNotFoundError: - raise HTTPException(404, f"No forensic data for container {container_id}") - if not rows: - raise HTTPException(404, f"Unknown container {container_id}") - return {"container_id": container_id, "objects": rows} - - -# ------------------------------------------------------------------ error index - - -@router.post("/megaeval/errors/search") -def error_search(req: ErrorSearchRequest): - store = get_mega_store() - _published_or_409(req.run_id) - art = store.artifacts(req.run_id) - return errors_mod.search_errors( - art.get("errors"), art.get("containers"), - error_types=req.error_types or None, filters=req.filters or None, - confidence_max=req.confidence_max, confidence_min=req.confidence_min, - risk_min=req.risk_min, severity_min=req.severity_min, - safety_only=req.safety_only, limit_containers=req.limit_containers) - - -# ------------------------------------------------------------------ compare / shift / why / similarity - - -@router.post("/megaeval/compare") -def compare(req: CompareRequest): - store = get_mega_store() - cand = _published_or_409(req.candidate_run_id) - base = _published_or_409(req.baseline_run_id) - return analysis.compare_runs(store, cand, base, req.policy or None) - - -@router.get("/megaeval/runs/{run_id}/shift") -def shift(run_id: str): - return analysis.distribution_shift(get_mega_store(), _published_or_409(run_id)) - - -@router.post("/megaeval/why") -def why(req: WhyRequest): - return analysis.why(get_mega_store(), _published_or_409(req.run_id), - req.filters or None, req.metric) - - -@router.post("/megaeval/similarity") -def similarity(req: SimilarityRequest): - try: - return analysis.similar_containers(get_mega_store(), _published_or_409(req.run_id), - req.container_id, req.filters or None, req.k) - except KeyError as e: - raise HTTPException(404, str(e)) - - -# ------------------------------------------------------------------ review sampling - - -@router.get("/megaeval/runs/{run_id}/review") -def review_state(run_id: str): - store = get_mega_store() - _published_or_409(run_id) - art = store.artifacts(run_id) - review = art.get("review") - if review is None: - return {"run_id": run_id, "planned": False, "executed": False} - slim = {k: v for k, v in review.items() if k != "plans"} - slim["planned"] = True - slim["strata"] = {name: plan["strata"] for name, plan in review["plans"].items()} - return slim - - -@router.post("/megaeval/runs/{run_id}/review/plan") -def review_plan(run_id: str, req: ReviewPlanRequest): - store = get_mega_store() - run = _published_or_409(run_id) - review = sampling.build_review_plan(store, run, target_n=req.target_n) - slim = {k: v for k, v in review.items() if k != "plans"} - slim["planned"] = True - slim["strata"] = {name: plan["strata"] for name, plan in review["plans"].items()} - return slim - - -@router.post("/megaeval/runs/{run_id}/review/execute") -def review_execute(run_id: str): - store = get_mega_store() - run = _published_or_409(run_id) - art = store.artifacts(run_id) - if art.get("review") is None: - sampling.build_review_plan(store, run) - review = sampling.execute_reviews(store, run) - slim = {k: v for k, v in review.items() if k != "plans"} - slim["planned"] = True - slim["strata"] = {name: plan["strata"] for name, plan in review["plans"].items()} - return slim - - -# ------------------------------------------------------------------ distributions (sketches) - - -@router.get("/megaeval/runs/{run_id}/distributions") -def distributions(run_id: str): - store = get_mega_store() - run = _published_or_409(run_id) - art = store.artifacts(run_id) - sk = art.get("sketches", {}) - out = {"run_id": run_id, "exact": False, - "note": "distributions are sketch-based (fixed-bin quantile histograms); " - "counts in the cube are exact"} - for name in ("confidence", "iou"): - hist = sk.get(name) - if hist is None: - continue - out[name] = { - "bins": hist.bins, "lo": hist.lo, "hi": hist.hi, - "counts": hist.counts.tolist(), - "percentiles": {q: round(hist.percentile(float(q)), 4) - for q in ("10", "25", "50", "75", "90", "99")}, - } - out["containers_hll_estimate"] = sk.get("container_hll_estimate") - out["containers_exact"] = run.headline.get("containers") - out["available_sketch_metrics"] = SKETCH_METRICS - return out diff --git a/hf-space/sensorflow/megaeval/cube.py b/hf-space/sensorflow/megaeval/cube.py deleted file mode 100644 index a524d16510ce8ad7747a36b44f9d37e87124c888..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/megaeval/cube.py +++ /dev/null @@ -1,270 +0,0 @@ -"""Metric cube: partial sufficient statistics -> reduce -> aggregate queries. - -The cube is the heart of aggregate-first evaluation. During a run, each worker -computes *partial* additive statistics per partition, grouped by the 9 cube -dimensions. Reducing = concatenating partials and summing per cell (map-side -combine + shuffle reduce in Spark terms). Queries NEVER scan raw records unless -they ask for something outside the cube (then the router honestly reports -source="scan"). - -Cell key: class x weather x lighting x road_type x scenario x sensor x - distance_band x speed_band x occlusion (per evaluation run) -Cell stats (all additive): n, tp, fp, fn, loc_err, anomalies, sum_iou, - sum_conf, sum_conf2, safety_n, safety_tp, reviewed, verified -Derived on demand: precision, recall, f1, mean_iou, anomaly_rate, - conf_mean, conf_std, safety_recall, review_coverage, verified_rate -""" - -from __future__ import annotations - -import hashlib -import json -import os -import threading -import time -from typing import Dict, List, Optional, Tuple - -import numpy as np -import pandas as pd - -from sensorflow.megaeval.population import DIMENSIONS, DIM_NAMES - -STAT_COLS = ["n", "tp", "fp", "fn", "loc_err", "anomalies", "sum_iou", - "sum_conf", "sum_conf2", "safety_n", "safety_tp", "reviewed", "verified"] - -DERIVED_METRICS = ["precision", "recall", "f1", "mean_iou", "anomaly_rate", - "conf_mean", "conf_std", "safety_recall", "review_coverage", - "verified_rate", "error_rate"] -COUNT_METRICS = ["n", "tp", "fp", "fn", "loc_err", "anomalies", "reviewed", "verified", - "safety_n", "safety_tp"] -SKETCH_METRICS = ["confidence_p50", "confidence_p10", "confidence_p90", - "iou_p50", "iou_p10", "iou_p90"] -ALL_METRICS = COUNT_METRICS + DERIVED_METRICS + SKETCH_METRICS - - -# ------------------------------------------------------------------ partial stats - - -def partial_stats(rows: pd.DataFrame) -> pd.DataFrame: - """Group one partition's contribution rows by the 9 dims and sum stat columns. - - `rows` must contain DIM_NAMES code columns plus STAT_COLS contribution columns - (e.g. a TP object contributes n=1, tp=1, sum_iou=iou; an FP row contributes - n=0, fp=1, sum_conf=conf). - """ - return rows.groupby(DIM_NAMES, as_index=False, observed=True, sort=False)[STAT_COLS].sum() - - -def reduce_partials(partials: List[pd.DataFrame]) -> pd.DataFrame: - """Reduce partial stats into the final cube (concat + groupby-sum).""" - if not partials: - return pd.DataFrame(columns=DIM_NAMES + STAT_COLS) - return (pd.concat(partials, ignore_index=True) - .groupby(DIM_NAMES, as_index=False, observed=True, sort=False)[STAT_COLS] - .sum()) - - -def derive_metrics(agg: pd.DataFrame) -> pd.DataFrame: - """Attach derived ratio metrics to an aggregated (already summed) frame.""" - out = agg.copy() - tp, fp, fn = out["tp"], out["fp"], out["fn"] - pp = tp + fp - with np.errstate(divide="ignore", invalid="ignore"): - out["precision"] = np.where(pp > 0, tp / pp, np.nan) - out["recall"] = np.where((tp + fn) > 0, tp / (tp + fn), np.nan) - p, r = out["precision"], out["recall"] - out["f1"] = np.where((p + r) > 0, 2 * p * r / (p + r), np.nan) - out["mean_iou"] = np.where(tp > 0, out["sum_iou"] / tp, np.nan) - out["anomaly_rate"] = np.where(out["n"] > 0, out["anomalies"] / out["n"], np.nan) - out["conf_mean"] = np.where(pp > 0, out["sum_conf"] / pp, np.nan) - var = np.where(pp > 1, - (out["sum_conf2"] - out["sum_conf"] ** 2 / pp.replace(0, 1)) / (pp - 1).replace(0, 1), - np.nan) - out["conf_std"] = np.sqrt(np.clip(var, 0, None)) - out["safety_recall"] = np.where(out["safety_n"] > 0, out["safety_tp"] / out["safety_n"], np.nan) - out["review_coverage"] = np.where(out["n"] > 0, out["reviewed"] / out["n"], np.nan) - out["verified_rate"] = np.where(out["reviewed"] > 0, out["verified"] / out["reviewed"], np.nan) - out["error_rate"] = np.where(out["n"] > 0, (fn + fp + out["loc_err"]) / out["n"], np.nan) - return out - - -# ------------------------------------------------------------------ persistence - - -def save_cube(path: str, cube: pd.DataFrame) -> None: - np.savez_compressed(path, **{c: cube[c].to_numpy() for c in cube.columns}) - - -def load_cube(path: str) -> pd.DataFrame: - with np.load(path) as z: - return pd.DataFrame({k: z[k] for k in z.files}) - - -# ------------------------------------------------------------------ query router - - -class QueryCache: - """Evaluation cache: key = hash(dataset+model+filters+group_by+metrics).""" - - def __init__(self, max_entries: int = 512): - self._data: Dict[str, Dict] = {} - self._lock = threading.Lock() - self.max_entries = max_entries - self.hits = 0 - self.misses = 0 - - @staticmethod - def key(dataset_version: str, model_version: str, filters: Dict, - group_by: List[str], metrics: List[str]) -> str: - blob = json.dumps({ - "d": dataset_version, "m": model_version, - "f": {k: sorted(v) for k, v in sorted((filters or {}).items())}, - "g": list(group_by or []), "x": sorted(metrics or []), - }, sort_keys=True) - return hashlib.sha256(blob.encode()).hexdigest()[:24] - - def get(self, key: str) -> Optional[Dict]: - with self._lock: - hit = self._data.get(key) - if hit is not None: - self.hits += 1 - else: - self.misses += 1 - return hit - - def put(self, key: str, value: Dict) -> None: - with self._lock: - if len(self._data) >= self.max_entries: - self._data.pop(next(iter(self._data))) - self._data[key] = value - - def invalidate(self) -> None: - with self._lock: - self._data.clear() - - def stats(self) -> Dict: - with self._lock: - total = self.hits + self.misses - return {"entries": len(self._data), "hits": self.hits, "misses": self.misses, - "hit_rate": round(self.hits / total, 4) if total else None} - - -def _codes_for(dim: str, values: List[str]) -> List[int]: - vocab = DIMENSIONS[dim] - out = [] - for v in values: - if v in vocab: - out.append(vocab.index(v)) - return out - - -def apply_filters(cube: pd.DataFrame, filters: Optional[Dict[str, List[str]]]) -> pd.DataFrame: - if not filters: - return cube - mask = np.ones(len(cube), dtype=bool) - for dim, values in filters.items(): - if dim not in DIM_NAMES: - continue - codes = _codes_for(dim, values if isinstance(values, list) else [values]) - mask &= cube[dim].isin(codes).to_numpy() - return cube[mask] - - -def aggregate(cube: pd.DataFrame, filters: Optional[Dict], group_by: Optional[List[str]], - metrics: Optional[List[str]], limit: int = 200) -> Tuple[List[Dict], int]: - """Aggregate-lookup path: filter cells, group, sum, derive. Never touches records.""" - sub = apply_filters(cube, filters) - group_by = [g for g in (group_by or []) if g in DIM_NAMES] - metrics = [m for m in (metrics or []) if m in ALL_METRICS and m not in SKETCH_METRICS] \ - or ["n", "precision", "recall", "f1", "mean_iou", "anomaly_rate"] - - if group_by: - agg = sub.groupby(group_by, as_index=False, observed=True, sort=False)[STAT_COLS].sum() - else: - sums = sub[STAT_COLS].sum() - agg = pd.DataFrame([sums]) - agg = derive_metrics(agg) - - cell_count = len(sub) - sort_metric = next((m for m in metrics if m in agg.columns), None) - if group_by and sort_metric: - agg = agg.sort_values("n", ascending=False) - agg = agg.head(max(1, min(limit, 2000))) - - rows: List[Dict] = [] - for _, r in agg.iterrows(): - row: Dict = {} - for g in group_by: - row[g] = DIMENSIONS[g][int(r[g])] - row["n"] = int(r["n"]) - for m in metrics: - if m == "n": - continue - v = r.get(m) - if v is None or (isinstance(v, float) and np.isnan(v)): - row[m] = None - elif m in COUNT_METRICS: - row[m] = int(v) - else: - row[m] = round(float(v), 6) - rows.append(row) - return rows, cell_count - - -class QueryRouter: - """Routes a query: cache -> cube aggregate -> (only if unavoidable) record scan. - - This is the OLAP-engine seam: callers never know how data is stored. - """ - - def __init__(self): - self.cache = QueryCache() - - def query(self, run: "object", cube: pd.DataFrame, sketches: Dict, - filters: Optional[Dict], metrics: Optional[List[str]], - group_by: Optional[List[str]], limit: int = 200, - scan_fn=None) -> Dict: - t0 = time.perf_counter() - metrics = metrics or [] - key = QueryCache.key(run.population_id, run.model_version, - filters or {}, group_by or [], metrics) - cached = self.cache.get(key) - if cached is not None: - return {**cached, - "meta": {**cached["meta"], "cache_hit": True, "source": "cache", - "latency_ms": round((time.perf_counter() - t0) * 1000, 3)}} - - sketch_wanted = [m for m in metrics if m in SKETCH_METRICS] - needs_scan = any(g not in DIM_NAMES for g in (group_by or [])) - - if needs_scan and scan_fn is not None: - rows, cell_count = scan_fn(filters, group_by, metrics, limit) - source, exact = "scan", True - else: - rows, cell_count = aggregate(cube, filters, group_by, metrics, limit) - source, exact = "cube", True - if sketch_wanted and not group_by and not filters: - # population-level percentile questions answered from sketches - for m in sketch_wanted: - metric_name, q = m.rsplit("_p", 1) - hist = sketches.get(metric_name) - if hist is not None and rows: - rows[0][m] = round(float(hist.percentile(float(q))), 6) - exact = False # percentiles are approximate - elif sketch_wanted: - exact = True # sketch metrics ignored on grouped/filtered queries - - latency = round((time.perf_counter() - t0) * 1000, 3) - result = { - "rows": rows, - "meta": { - "source": source, - "cache_hit": False, - "latency_ms": latency, - "cells_touched": cell_count, - "exact": exact, - "approximate_fields": sketch_wanted if not exact else [], - }, - } - self.cache.put(key, result) - return result diff --git a/hf-space/sensorflow/megaeval/errors.py b/hf-space/sensorflow/megaeval/errors.py deleted file mode 100644 index 90e2b3a0682d02a5b607af0429d3d5b71dc634da..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/megaeval/errors.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Error index: fast multi-criteria search over evaluation errors. - -The index is a columnar table (one row per error) built during the run: - error_id, object_id, container_id, error_type, severity, confidence, - risk_score, safety_critical, sensor_disagree, + the 9 cube dimensions. - -Searches return the *worst-N containers* (aggregated), never raw record dumps — -this backs the Investigation UI. Individual exemplar errors are capped. -""" - -from __future__ import annotations - -from typing import Dict, List, Optional - -import numpy as np -import pandas as pd - -from sensorflow.megaeval.population import DIMENSIONS, DIM_NAMES -from sensorflow.megaeval.runs import ERROR_TYPES - - -def search_errors( - errors: pd.DataFrame, - containers: pd.DataFrame, - error_types: Optional[List[str]] = None, - filters: Optional[Dict[str, List[str]]] = None, - confidence_max: Optional[float] = None, - confidence_min: Optional[float] = None, - risk_min: Optional[float] = None, - severity_min: Optional[float] = None, - safety_only: bool = False, - limit_containers: int = 25, - limit_examples: int = 50, -) -> Dict: - if errors is None or not len(errors): - return {"matched_errors": 0, "worst_containers": [], "examples": [], - "by_type": {}} - mask = np.ones(len(errors), dtype=bool) - if error_types: - codes = [ERROR_TYPES.index(t) for t in error_types if t in ERROR_TYPES] - mask &= errors["error_type"].isin(codes).to_numpy() - for dim, values in (filters or {}).items(): - if dim not in DIM_NAMES: - continue - vocab = DIMENSIONS[dim] - codes = [vocab.index(v) for v in (values if isinstance(values, list) else [values]) - if v in vocab] - mask &= errors[dim].isin(codes).to_numpy() - if confidence_max is not None: - mask &= errors["confidence"].to_numpy() <= confidence_max - if confidence_min is not None: - mask &= errors["confidence"].to_numpy() >= confidence_min - if risk_min is not None: - mask &= errors["risk_score"].to_numpy() >= risk_min - if severity_min is not None: - mask &= errors["severity"].to_numpy() >= severity_min - if safety_only: - mask &= errors["safety_critical"].to_numpy().astype(bool) - - sub = errors[mask] - by_type = {ERROR_TYPES[int(c)]: int(n) - for c, n in sub["error_type"].value_counts().items()} - - # aggregate to worst containers: rank by count x mean risk - grp = sub.groupby("container_id", as_index=False).agg( - error_count=("error_id", "size"), mean_risk=("risk_score", "mean"), - max_severity=("severity", "max"), safety_hits=("safety_critical", "sum")) - grp["rank_score"] = grp["error_count"] * grp["mean_risk"] - grp = grp.sort_values("rank_score", ascending=False).head(limit_containers) - cmeta = containers.set_index("container_id") if containers is not None else None - worst = [] - for _, r in grp.iterrows(): - cid = int(r["container_id"]) - item = {"container_id": cid, "error_count": int(r["error_count"]), - "mean_risk": round(float(r["mean_risk"]), 4), - "max_severity": round(float(r["max_severity"]), 4), - "safety_hits": int(r["safety_hits"])} - if cmeta is not None and cid in cmeta.index: - row = cmeta.loc[cid] - for dim in ("weather", "lighting", "road_type", "scenario"): - item[dim] = DIMENSIONS[dim][int(row[dim])] - item["n_objects"] = int(row["n_objects"]) - item["risk_score"] = round(float(row["risk_score"]), 4) - worst.append(item) - - ex = sub.sort_values("risk_score", ascending=False).head(limit_examples) - examples = [] - for _, r in ex.iterrows(): - examples.append({ - "error_id": int(r["error_id"]), - "annotation_id": (f"obj-{int(r['object_id'])}" if r["object_id"] >= 0 - else f"fp-{int(r['error_id'])}"), - "container_id": int(r["container_id"]), - "error_type": ERROR_TYPES[int(r["error_type"])], - "class": DIMENSIONS["class"][int(r["class"])], - "severity": round(float(r["severity"]), 4), - "confidence": round(float(r["confidence"]), 4), - "risk_score": round(float(r["risk_score"]), 4), - "safety_critical": bool(r["safety_critical"]), - "scenario": DIMENSIONS["scenario"][int(r["scenario"])], - "lighting": DIMENSIONS["lighting"][int(r["lighting"])], - "weather": DIMENSIONS["weather"][int(r["weather"])], - }) - - return {"matched_errors": int(len(sub)), "by_type": by_type, - "worst_containers": worst, "examples": examples} diff --git a/hf-space/sensorflow/megaeval/population.py b/hf-space/sensorflow/megaeval/population.py deleted file mode 100644 index 409c789f3e852b987ce10343c594db02490f44cf..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/megaeval/population.py +++ /dev/null @@ -1,281 +0,0 @@ -"""Immutable synthetic populations, generated vectorized and stored partitioned. - -Storage layout (single-node stand-in for an Iceberg/parquet lakehouse — no parquet -engine is installed in this venv, so partitions are compressed npz column files): - - runs/megaeval/populations/{pop_id}/ - meta.json dataset card: dims vocab, counts (exact), train mix, lineage - part-0000.npz ... column arrays partitioned by container_id % NUM_PARTITIONS - -Every object row carries: - - 9 low-cardinality dimension codes (the cube dimensions) - - a model-independent `difficulty` in [0,1] (occlusion/distance/lighting/weather/size) - - `safety_critical` flag (VRUs in high-risk scenarios) - -The "training mix" is a deliberately shifted joint distribution over -(class, weather, lighting) used by shift detection: training saw less -night/rain/fog than this evaluation population. -""" - -from __future__ import annotations - -import json -import os -import threading -import uuid -from datetime import datetime, timezone -from typing import Dict, List, Optional, Tuple - -import numpy as np - -MEGA_ROOT = os.path.join("runs", "megaeval") -NUM_PARTITIONS = 32 - -DIMENSIONS: Dict[str, List[str]] = { - "class": ["vehicle", "pedestrian", "cyclist", "motorcycle", "truck", "bus"], - "weather": ["clear", "rain", "fog", "snow"], - "lighting": ["day", "night", "dusk"], - "road_type": ["highway", "urban", "rural", "intersection"], - "scenario": ["nominal", "near_miss", "occlusion_heavy", "night_glare", - "extreme_ttc", "sensor_degraded"], - "sensor": ["lidar", "camera", "fused"], - "distance_band": ["0-20m", "20-50m", "50-100m", "100m+"], - "speed_band": ["stationary", "slow", "medium", "fast"], - "occlusion": ["none", "partial", "heavy"], -} -DIM_NAMES = list(DIMENSIONS.keys()) -CONTAINER_DIMS = ["weather", "lighting", "road_type", "scenario"] -OBJECT_DIMS = ["class", "sensor", "distance_band", "speed_band", "occlusion"] - - -def _now() -> str: - return datetime.now(timezone.utc).isoformat() - - -def pop_dir(pop_id: str) -> str: - return os.path.join(MEGA_ROOT, "populations", pop_id) - - -def set_mega_root(path: str) -> None: - """Test hook: relocate all megaeval storage (and clear partition caches).""" - global MEGA_ROOT - MEGA_ROOT = str(path) - with _CACHE_LOCK: - _PART_CACHE.clear() - - -# ------------------------------------------------------------------ generation - - -def _sample_container_dims(rng: np.random.Generator, n: int) -> Dict[str, np.ndarray]: - """Correlated container-level context (weather/lighting/road/scenario).""" - weather = rng.choice(4, size=n, p=[0.62, 0.20, 0.11, 0.07]).astype(np.int8) - lighting = rng.choice(3, size=n, p=[0.60, 0.28, 0.12]).astype(np.int8) - road = rng.choice(4, size=n, p=[0.30, 0.38, 0.14, 0.18]).astype(np.int8) - scenario = rng.choice(6, size=n, p=[0.66, 0.09, 0.09, 0.06, 0.05, 0.05]).astype(np.int8) - # correlations: night_glare scenario forces night; fog/snow raise sensor_degraded odds - scenario_names = DIMENSIONS["scenario"] - lighting[scenario == scenario_names.index("night_glare")] = 1 # night - bad_wx = (weather >= 2) - flip = bad_wx & (rng.random(n) < 0.18) - scenario[flip] = scenario_names.index("sensor_degraded") - return {"weather": weather, "lighting": lighting, "road_type": road, "scenario": scenario} - - -_CLASS_MIX_BY_ROAD = np.array([ - # vehicle, pedestrian, cyclist, motorcycle, truck, bus - [0.62, 0.02, 0.01, 0.05, 0.24, 0.06], # highway - [0.44, 0.26, 0.12, 0.06, 0.07, 0.05], # urban - [0.55, 0.08, 0.06, 0.08, 0.18, 0.05], # rural - [0.40, 0.30, 0.14, 0.05, 0.06, 0.05], # intersection -]) - -_SPEED_MIX_BY_ROAD = np.array([ - [0.02, 0.08, 0.35, 0.55], # highway - [0.22, 0.44, 0.28, 0.06], # urban - [0.10, 0.30, 0.40, 0.20], # rural - [0.30, 0.45, 0.20, 0.05], # intersection -]) - - -def _object_difficulty(codes: Dict[str, np.ndarray], rng: np.random.Generator) -> np.ndarray: - """Model-independent difficulty in [0,1] — the physical hardness of each object.""" - n = codes["class"].shape[0] - d = np.full(n, 0.12, dtype=np.float32) - d += np.array([0.00, 0.14, 0.12, 0.10, 0.02, 0.02], dtype=np.float32)[codes["class"]] - d += np.array([0.00, 0.08, 0.12, 0.10], dtype=np.float32)[codes["weather"]] - d += np.array([0.00, 0.12, 0.05], dtype=np.float32)[codes["lighting"]] - d += np.array([0.00, 0.10, 0.24], dtype=np.float32)[codes["occlusion"]] - d += np.array([0.00, 0.05, 0.14, 0.26], dtype=np.float32)[codes["distance_band"]] - d += np.array([0.00, 0.04, 0.08, 0.06, 0.07, 0.12], dtype=np.float32)[codes["scenario"]] - d += rng.normal(0.0, 0.05, size=n).astype(np.float32) - return np.clip(d, 0.01, 0.99) - - -def generate_population( - name: str = "mega-perception", - num_objects: int = 320_000, - seed: int = 42, - avg_objects_per_container: float = 12.0, -) -> Dict: - """Generate + persist a partitioned population. Vectorized; ~seconds for 320k.""" - rng = np.random.default_rng(seed) - num_containers = max(64, int(num_objects / avg_objects_per_container)) - - cdims = _sample_container_dims(rng, num_containers) - per_container = np.clip(rng.poisson(avg_objects_per_container, size=num_containers), - 3, 48).astype(np.int64) - total = int(per_container.sum()) - container_id = np.repeat(np.arange(num_containers, dtype=np.int64), per_container) - - codes: Dict[str, np.ndarray] = {k: cdims[k][container_id] for k in CONTAINER_DIMS} - - road_obj = codes["road_type"].astype(np.int64) - u = rng.random(total) - cls_cdf = np.cumsum(_CLASS_MIX_BY_ROAD, axis=1)[road_obj] - codes["class"] = (u[:, None] > cls_cdf).sum(axis=1).astype(np.int8) - u2 = rng.random(total) - spd_cdf = np.cumsum(_SPEED_MIX_BY_ROAD, axis=1)[road_obj] - codes["speed_band"] = (u2[:, None] > spd_cdf).sum(axis=1).astype(np.int8) - codes["sensor"] = rng.choice(3, size=total, p=[0.38, 0.22, 0.40]).astype(np.int8) - codes["distance_band"] = rng.choice(4, size=total, p=[0.24, 0.40, 0.25, 0.11]).astype(np.int8) - - occ_p = np.tile(np.array([0.62, 0.28, 0.10]), (total, 1)) - heavy_scene = codes["scenario"] == DIMENSIONS["scenario"].index("occlusion_heavy") - occ_p[heavy_scene] = [0.20, 0.42, 0.38] - occ_cdf = np.cumsum(occ_p, axis=1) - codes["occlusion"] = (rng.random(total)[:, None] > occ_cdf).sum(axis=1).astype(np.int8) - - difficulty = _object_difficulty(codes, rng) - vru = np.isin(codes["class"], [1, 2, 3]) - risky_scene = np.isin(codes["scenario"], [1, 4]) # near_miss, extreme_ttc - near = codes["distance_band"] <= 1 - safety_critical = (vru & (risky_scene | (near & (rng.random(total) < 0.15)))).astype(np.uint8) - - object_id = np.arange(total, dtype=np.int64) - pop_id = f"pop-{uuid.uuid4().hex[:10]}" - d = pop_dir(pop_id) - os.makedirs(d, exist_ok=True) - - part_of = (container_id % NUM_PARTITIONS).astype(np.int64) - order = np.argsort(part_of, kind="stable") - part_sorted = part_of[order] - bounds = np.searchsorted(part_sorted, np.arange(NUM_PARTITIONS + 1)) - partition_meta = [] - for p in range(NUM_PARTITIONS): - sl = order[bounds[p]:bounds[p + 1]] - cols = {k: codes[k][sl] for k in DIM_NAMES} - cols["object_id"] = object_id[sl] - cols["container_id"] = container_id[sl] - cols["difficulty"] = difficulty[sl] - cols["safety_critical"] = safety_critical[sl] - np.savez_compressed(os.path.join(d, f"part-{p:04d}.npz"), **cols) - partition_meta.append({"partition": p, "rows": int(sl.size)}) - - # exact per-dimension counts (dataset card) - dim_counts = { - k: {DIMENSIONS[k][i]: int(c) for i, c in enumerate(np.bincount(codes[k], minlength=len(DIMENSIONS[k])))} - for k in DIM_NAMES - } - - train_mix = _training_mix(codes, rng) - - meta = { - "population_id": pop_id, - "name": name, - "created_at": _now(), - "seed": seed, - "num_objects": total, - "num_containers": num_containers, - "num_partitions": NUM_PARTITIONS, - "avg_objects_per_container": round(total / num_containers, 2), - "dimensions": DIMENSIONS, - "dim_counts": dim_counts, - "partitions": partition_meta, - "safety_critical_count": int(safety_critical.sum()), - "train_mix": train_mix, - } - with open(os.path.join(d, "meta.json"), "w") as f: - json.dump(meta, f) - return meta - - -def _training_mix(codes: Dict[str, np.ndarray], rng: np.random.Generator) -> List[Dict]: - """Joint (class, weather, lighting) proportions the *training* set had. - - Derived from the eval mix with deliberate, deterministic under-representation of - hard conditions (night, rain/fog/snow, VRU at night) so shift detection has - honest signal: shift = eval share vs train share of the same cohort. - """ - n = codes["class"].shape[0] - key = (codes["class"].astype(np.int64) * 100 - + codes["weather"].astype(np.int64) * 10 - + codes["lighting"].astype(np.int64)) - uniq, counts = np.unique(key, return_counts=True) - eval_share = counts / n - - cls, wx, lt = uniq // 100, (uniq // 10) % 10, uniq % 10 - factor = np.ones(uniq.shape) - factor[lt == 1] *= 0.55 # trained on less night - factor[wx >= 1] *= 0.70 # less rain/fog/snow - factor[(np.isin(cls, [1, 2])) & (lt == 1)] *= 0.75 # far fewer night VRUs - train_share = eval_share * factor - train_share = train_share / train_share.sum() - - out = [] - for i in range(uniq.size): - out.append({ - "class": DIMENSIONS["class"][int(cls[i])], - "weather": DIMENSIONS["weather"][int(wx[i])], - "lighting": DIMENSIONS["lighting"][int(lt[i])], - "train_share": float(train_share[i]), - "eval_share": float(eval_share[i]), - "eval_count": int(counts[i]), - }) - return out - - -# ------------------------------------------------------------------ loading - - -_PART_CACHE: Dict[Tuple[str, int], Dict[str, np.ndarray]] = {} -_CACHE_LOCK = threading.Lock() - - -def load_meta(pop_id: str) -> Optional[Dict]: - path = os.path.join(pop_dir(pop_id), "meta.json") - if not os.path.exists(path): - return None - with open(path) as f: - return json.load(f) - - -def list_populations() -> List[Dict]: - base = os.path.join(MEGA_ROOT, "populations") - if not os.path.isdir(base): - return [] - metas = [] - for pid in sorted(os.listdir(base)): - m = load_meta(pid) - if m: - metas.append(m) - return metas - - -def load_partition(pop_id: str, partition: int) -> Dict[str, np.ndarray]: - key = (pop_id, partition) - with _CACHE_LOCK: - if key in _PART_CACHE: - return _PART_CACHE[key] - path = os.path.join(pop_dir(pop_id), f"part-{partition:04d}.npz") - with np.load(path) as z: - cols = {k: z[k] for k in z.files} - with _CACHE_LOCK: - if len(_PART_CACHE) > 96: # bounded partition cache - _PART_CACHE.clear() - _PART_CACHE[key] = cols - return cols - - -def partition_of_container(container_id: int) -> int: - return int(container_id) % NUM_PARTITIONS diff --git a/hf-space/sensorflow/megaeval/runs.py b/hf-space/sensorflow/megaeval/runs.py deleted file mode 100644 index 1e454ca609b8122abab25673521bf2646dd4aa55..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/megaeval/runs.py +++ /dev/null @@ -1,704 +0,0 @@ -"""Evaluation runs: first-class async jobs producing cubes, error indexes and sketches. - -Lifecycle: create -> queued -> running (workers over partitions, partial stats) - -> reducing -> materializing -> published (or failed) - -Each worker processes one population partition (Spark-executor stand-in): - 1. vectorized model simulation (deterministic given lineage seed) - 2. partial sufficient statistics grouped by the 9 cube dims - 3. per-object outputs persisted for forensic drill-down / honest scan fallback - 4. error rows + sketch updates + container partial aggregates - -Reduce = concat partials + groupby-sum (cube.reduce_partials). -""" - -from __future__ import annotations - -import hashlib -import json -import os -import platform -import threading -import time -import uuid -from concurrent.futures import ThreadPoolExecutor -from datetime import datetime, timezone -from typing import Dict, List, Optional - -import numpy as np -import pandas as pd - -from sensorflow.megaeval import cube as cube_mod -from sensorflow.megaeval import population as pop_mod -from sensorflow.megaeval.population import DIMENSIONS, DIM_NAMES -from sensorflow.megaeval.sketches import HyperLogLog, QuantileHistogram, Reservoir - -ERROR_TYPES = ["FN", "FP", "LOCALIZATION", "ANOMALY", "LOW_CONF"] -CONTAINER_STATUS = ["ok", "warn", "critical"] -IOU_MATCH_THRESHOLD = 0.5 -EVALUATOR_CODE_VERSION = "megaeval-1.0.0" -METRIC_VERSION = "metrics-1.0" - - -def _now() -> str: - return datetime.now(timezone.utc).isoformat() - - -def run_dir(run_id: str) -> str: - return os.path.join(pop_mod.MEGA_ROOT, "runs", run_id) - - -# ------------------------------------------------------------------ model profiles - - -def model_params(model_version: str, overrides: Optional[Dict] = None) -> Dict: - """Deterministic model skill profile derived from the version string. - - Overrides let tests/demos inject regressions (e.g. worse night detection). - """ - h = int(hashlib.sha256(model_version.encode()).hexdigest()[:8], 16) - r = np.random.default_rng(h) - params = { - "base_detect": 0.972 + float(r.uniform(-0.01, 0.015)), - "difficulty_weight": 0.18, - "night_penalty": 0.030 + float(r.uniform(0, 0.02)), - "rain_penalty": 0.020 + float(r.uniform(0, 0.015)), - "occlusion_penalty": 0.050, - "vru_penalty": 0.020 + float(r.uniform(0, 0.015)), - "far_penalty": 0.045, - "iou_base": 0.875 + float(r.uniform(-0.01, 0.015)), - "iou_difficulty": 0.28, - "fp_per_container": 0.30 + float(r.uniform(0, 0.08)), - "anomaly_base": 0.018, - "low_conf_threshold": 0.45, - } - params.update(overrides or {}) - return params - - -# ------------------------------------------------------------------ run entity - - -class EvaluationRun: - def __init__(self, population_id: str, model_version: str, - overrides: Optional[Dict] = None, seed: Optional[int] = None, - worker_delay_s: float = 0.0, workers: int = 4, - label_version: str = "labels-v1", - threshold_config: Optional[Dict] = None, - sampling_config: Optional[Dict] = None): - self.run_id = f"eval-{uuid.uuid4().hex[:10]}" - self.population_id = population_id - self.model_version = model_version - self.overrides = overrides or {} - self.seed = seed if seed is not None else int( - hashlib.sha256(f"{population_id}|{model_version}|{label_version}".encode() - ).hexdigest()[:8], 16) - self.worker_delay_s = worker_delay_s - self.workers = workers - self.status = "created" - self.error: Optional[str] = None - self.created_at = _now() - self.started_at: Optional[str] = None - self.published_at: Optional[str] = None - self.partitions_total = 0 - self.partitions_done = 0 - self.objects_total = 0 - self.objects_processed = 0 - self.throughput_objs_per_s = 0.0 - self.eta_s: Optional[float] = None - self.headline: Dict = {} - self.per_class: Dict = {} - self.lineage = { - "evaluation_id": self.run_id, - "dataset_version": population_id, - "model_version": model_version, - "model_checkpoint": f"ckpt-{hashlib.sha256(model_version.encode()).hexdigest()[:10]}", - "label_version": label_version, - "evaluator_code_version": EVALUATOR_CODE_VERSION, - "metric_version": METRIC_VERSION, - "threshold_config": {"iou_match": IOU_MATCH_THRESHOLD, - **(threshold_config or {})}, - "sampling_config": sampling_config or {"method": "stratified_risk_weighted", - "target_n": 1500}, - "seed": self.seed, - "hardware": f"{platform.machine()}/{platform.system()} single-node", - "timestamp": self.created_at, - } - - def progress_dict(self) -> Dict: - pct = (self.partitions_done / self.partitions_total * 100) if self.partitions_total else 0.0 - return { - "run_id": self.run_id, - "population_id": self.population_id, - "model_version": self.model_version, - "status": self.status, - "percent": round(pct, 2), - "partitions_done": self.partitions_done, - "partitions_total": self.partitions_total, - "objects_processed": int(self.objects_processed), - "objects_total": int(self.objects_total), - "workers": self.workers, - "throughput_objs_per_s": round(self.throughput_objs_per_s, 1), - "eta_s": round(self.eta_s, 1) if self.eta_s is not None else None, - "error": self.error, - } - - def to_dict(self) -> Dict: - return { - **self.progress_dict(), - "created_at": self.created_at, - "started_at": self.started_at, - "published_at": self.published_at, - "lineage": self.lineage, - "headline": self.headline, - "per_class": self.per_class, - "overrides": self.overrides, - } - - -# ------------------------------------------------------------------ partition evaluation - - -def evaluate_partition(pop_id: str, partition: int, params: Dict, seed: int) -> Dict: - """Simulate the model on one partition; return partials + per-object outputs. - - Fully vectorized and deterministic for (lineage seed, partition). - """ - cols = pop_mod.load_partition(pop_id, partition) - n = cols["object_id"].shape[0] - rng = np.random.default_rng((seed * 1_000_003 + partition) % (2**63)) - - difficulty = cols["difficulty"].astype(np.float64) - p_det = params["base_detect"] - difficulty * params["difficulty_weight"] - p_det -= params["night_penalty"] * (cols["lighting"] == 1) - p_det -= params["rain_penalty"] * (cols["weather"] >= 1) - p_det -= params["occlusion_penalty"] * (cols["occlusion"] == 2) - p_det -= params["vru_penalty"] * np.isin(cols["class"], [1, 2, 3]) - p_det -= params["far_penalty"] * (cols["distance_band"] == 3) - p_det = np.clip(p_det, 0.02, 0.995) - detected = rng.random(n) < p_det - - iou = np.zeros(n, dtype=np.float64) - iou[detected] = np.clip( - rng.normal(params["iou_base"] - difficulty[detected] * params["iou_difficulty"], 0.07), - 0.02, 0.985) - tp = detected & (iou >= IOU_MATCH_THRESHOLD) - loc = detected & ~tp - - conf = np.zeros(n, dtype=np.float64) - conf[detected] = np.clip( - 0.55 * iou[detected] + 0.25 * (1 - difficulty[detected]) - + rng.normal(0.05, 0.08, size=int(detected.sum())), 0.01, 0.999) - - anomaly_p = params["anomaly_base"] * (1 + 2.0 * difficulty - + 0.8 * (cols["scenario"] >= 4)) - anomaly = rng.random(n) < anomaly_p - sensor_disagree = rng.random(n) < (0.04 + 0.25 * (cols["scenario"] == 5)) - - safety = cols["safety_critical"].astype(bool) - - # ---- spurious FPs per container - uniq_containers, first_idx = np.unique(cols["container_id"], return_index=True) - ncont = uniq_containers.size - wx_c = cols["weather"][first_idx] - lt_c = cols["lighting"][first_idx] - lam = params["fp_per_container"] * (1 + 0.5 * (wx_c >= 1) + 0.6 * (lt_c == 1)) - fp_counts = rng.poisson(lam) - total_fp = int(fp_counts.sum()) - fp_container = np.repeat(uniq_containers, fp_counts) - fp_src = np.repeat(first_idx, fp_counts) - fp_cols = {k: cols[k][fp_src] for k in pop_mod.CONTAINER_DIMS} - fp_cols["class"] = rng.choice(6, size=total_fp, - p=[0.42, 0.18, 0.10, 0.08, 0.14, 0.08]).astype(np.int8) - fp_cols["sensor"] = rng.choice(3, size=total_fp).astype(np.int8) - fp_cols["distance_band"] = rng.choice(4, size=total_fp, p=[0.15, 0.35, 0.30, 0.20]).astype(np.int8) - fp_cols["speed_band"] = rng.choice(4, size=total_fp).astype(np.int8) - fp_cols["occlusion"] = rng.choice(3, size=total_fp, p=[0.5, 0.3, 0.2]).astype(np.int8) - fp_conf = np.clip(rng.beta(2.0, 4.5, size=total_fp) + 0.05, 0.01, 0.99) - - # ---- contribution rows -> partial stats - gt = pd.DataFrame({k: cols[k] for k in DIM_NAMES}) - gt["n"] = 1 - gt["tp"] = tp.astype(np.int64) - gt["fp"] = loc.astype(np.int64) # badly localized box counts as FP too - gt["fn"] = (~tp).astype(np.int64) - gt["loc_err"] = loc.astype(np.int64) - gt["anomalies"] = anomaly.astype(np.int64) - gt["sum_iou"] = np.where(tp, iou, 0.0) - gt["sum_conf"] = np.where(detected, conf, 0.0) - gt["sum_conf2"] = np.where(detected, conf**2, 0.0) - gt["safety_n"] = safety.astype(np.int64) - gt["safety_tp"] = (safety & tp).astype(np.int64) - gt["reviewed"] = 0 - gt["verified"] = 0 - - fpf = pd.DataFrame({k: fp_cols[k] for k in DIM_NAMES}) - fpf["n"] = 0 - fpf["tp"] = 0 - fpf["fp"] = 1 - fpf["fn"] = 0 - fpf["loc_err"] = 0 - fpf["anomalies"] = 0 - fpf["sum_iou"] = 0.0 - fpf["sum_conf"] = fp_conf - fpf["sum_conf2"] = fp_conf**2 - fpf["safety_n"] = 0 - fpf["safety_tp"] = 0 - fpf["reviewed"] = 0 - fpf["verified"] = 0 - - partial = cube_mod.partial_stats(pd.concat([gt, fpf], ignore_index=True)) - - # ---- error rows (error index) - err_frames = [] - fn_mask = ~detected - for etype, mask, sev_base in (("FN", fn_mask, 0.75), ("LOCALIZATION", loc, 0.55)): - idx = np.where(mask)[0] - if idx.size == 0: - continue - code = ERROR_TYPES.index(etype) - err_frames.append(pd.DataFrame({ - "object_id": cols["object_id"][idx], - "container_id": cols["container_id"][idx], - "error_type": np.full(idx.size, code, dtype=np.int8), - "severity": np.clip(sev_base + 0.25 * safety[idx] + 0.15 * difficulty[idx], 0, 1), - "confidence": conf[idx], - "safety_critical": safety[idx].astype(np.uint8), - "sensor_disagree": sensor_disagree[idx].astype(np.uint8), - **{k: cols[k][idx] for k in DIM_NAMES}, - })) - an_idx = np.where(anomaly & tp)[0] - if an_idx.size: - err_frames.append(pd.DataFrame({ - "object_id": cols["object_id"][an_idx], - "container_id": cols["container_id"][an_idx], - "error_type": np.full(an_idx.size, ERROR_TYPES.index("ANOMALY"), dtype=np.int8), - "severity": np.clip(0.4 + 0.3 * difficulty[an_idx], 0, 1), - "confidence": conf[an_idx], - "safety_critical": safety[an_idx].astype(np.uint8), - "sensor_disagree": sensor_disagree[an_idx].astype(np.uint8), - **{k: cols[k][an_idx] for k in DIM_NAMES}, - })) - lc_idx = np.where(tp & (conf < params["low_conf_threshold"]))[0] - if lc_idx.size: - err_frames.append(pd.DataFrame({ - "object_id": cols["object_id"][lc_idx], - "container_id": cols["container_id"][lc_idx], - "error_type": np.full(lc_idx.size, ERROR_TYPES.index("LOW_CONF"), dtype=np.int8), - "severity": np.clip(0.3 + 0.2 * difficulty[lc_idx], 0, 1), - "confidence": conf[lc_idx], - "safety_critical": safety[lc_idx].astype(np.uint8), - "sensor_disagree": sensor_disagree[lc_idx].astype(np.uint8), - **{k: cols[k][lc_idx] for k in DIM_NAMES}, - })) - if total_fp: - err_frames.append(pd.DataFrame({ - "object_id": np.full(total_fp, -1, dtype=np.int64), - "container_id": fp_container, - "error_type": np.full(total_fp, ERROR_TYPES.index("FP"), dtype=np.int8), - "severity": np.clip(0.45 + 0.4 * fp_conf, 0, 1), - "confidence": fp_conf, - "safety_critical": np.zeros(total_fp, dtype=np.uint8), - "sensor_disagree": (rng.random(total_fp) < 0.08).astype(np.uint8), - **{k: fp_cols[k] for k in DIM_NAMES}, - })) - errors = (pd.concat(err_frames, ignore_index=True) if err_frames - else pd.DataFrame()) - - # ---- container partial aggregates - cdf = pd.DataFrame({ - "container_id": cols["container_id"], - **{k: cols[k] for k in pop_mod.CONTAINER_DIMS}, - "n_objects": 1, - "tp": tp.astype(np.int64), - "fp": loc.astype(np.int64), - "fn": (~tp).astype(np.int64), - "anomalies": anomaly.astype(np.int64), - "sum_iou": np.where(tp, iou, 0.0), - "safety_n": safety.astype(np.int64), - }) - containers = cdf.groupby(["container_id"] + pop_mod.CONTAINER_DIMS, - as_index=False, observed=True).sum() - if total_fp: - fp_per = pd.Series(fp_container).value_counts() - containers["fp"] = containers["fp"] + containers["container_id"].map(fp_per).fillna(0).astype(np.int64) - - # ---- sketches (per partition, merged at reduce) - conf_hist = QuantileHistogram(0, 1, 64) - conf_hist.add(np.concatenate([conf[detected], fp_conf])) - iou_hist = QuantileHistogram(0, 1, 64) - iou_hist.add(iou[tp]) - hll = HyperLogLog(12) - hll.add_ids(uniq_containers) - reservoir = Reservoir(k=100, seed=partition) - if len(errors): - reservoir.add_ids(errors["object_id"].to_numpy()[errors["object_id"].to_numpy() >= 0][:2000]) - - # ---- per-object outputs (forensic view + scan fallback + brute-force tests) - obj_out = { - "object_id": cols["object_id"], - "container_id": cols["container_id"], - "detected": detected.astype(np.uint8), - "tp": tp.astype(np.uint8), - "iou": iou.astype(np.float32), - "confidence": conf.astype(np.float32), - "anomaly": anomaly.astype(np.uint8), - "sensor_disagree": sensor_disagree.astype(np.uint8), - } - fp_out = { - "container_id": fp_container, - "confidence": fp_conf.astype(np.float32), - **{k: fp_cols[k] for k in DIM_NAMES}, - } - - return {"partial": partial, "errors": errors, "containers": containers, - "conf_hist": conf_hist, "iou_hist": iou_hist, "hll": hll, - "reservoir": reservoir, "objects": obj_out, "fps": fp_out, - "rows": n} - - -# ------------------------------------------------------------------ store - - -class MegaStore: - """Registry of populations + runs, with lazily-loaded run artifacts.""" - - def __init__(self): - self.lock = threading.RLock() - self.runs: Dict[str, EvaluationRun] = {} - self._artifacts: Dict[str, Dict] = {} - self.router = cube_mod.QueryRouter() - self._load_persisted_runs() - - def _load_persisted_runs(self) -> None: - base = os.path.join(pop_mod.MEGA_ROOT, "runs") - if not os.path.isdir(base): - return - for rid in sorted(os.listdir(base)): - meta_path = os.path.join(base, rid, "run.json") - if not os.path.exists(meta_path): - continue - try: - with open(meta_path) as f: - d = json.load(f) - run = EvaluationRun.__new__(EvaluationRun) - run.__dict__.update(d["state"]) - self.runs[run.run_id] = run - except Exception: - continue - - def save_run(self, run: EvaluationRun) -> None: - d = run_dir(run.run_id) - os.makedirs(d, exist_ok=True) - state = {k: v for k, v in run.__dict__.items() if not k.startswith("_")} - with open(os.path.join(d, "run.json"), "w") as f: - json.dump({"state": state}, f) - - # ---- artifacts - - def artifacts(self, run_id: str) -> Dict: - with self.lock: - if run_id in self._artifacts: - return self._artifacts[run_id] - d = run_dir(run_id) - art: Dict = {} - cube_path = os.path.join(d, "cube.npz") - if os.path.exists(cube_path): - art["cube"] = cube_mod.load_cube(cube_path) - err_path = os.path.join(d, "errors.npz") - if os.path.exists(err_path): - with np.load(err_path) as z: - art["errors"] = pd.DataFrame({k: z[k] for k in z.files}) - cont_path = os.path.join(d, "containers.npz") - if os.path.exists(cont_path): - with np.load(cont_path) as z: - art["containers"] = pd.DataFrame({k: z[k] for k in z.files}) - emb_path = os.path.join(d, "embeddings.npz") - if os.path.exists(emb_path): - with np.load(emb_path) as z: - art["emb_ids"] = z["container_id"] - art["emb"] = z["emb"] - sk_path = os.path.join(d, "sketches.npz") - if os.path.exists(sk_path): - with np.load(sk_path) as z: - art["sketches"] = { - "confidence": QuantileHistogram.from_arrays(0, 1, z["conf_hist"]), - "iou": QuantileHistogram.from_arrays(0, 1, z["iou_hist"]), - "container_hll_estimate": float(z["hll_estimate"]), - "exemplar_ids": z["exemplars"].tolist(), - } - rev_path = os.path.join(d, "review.json") - if os.path.exists(rev_path): - with open(rev_path) as f: - art["review"] = json.load(f) - with self.lock: - if len(self._artifacts) > 6: - self._artifacts.pop(next(iter(self._artifacts))) - self._artifacts[run_id] = art - return art - - def invalidate_artifacts(self, run_id: str) -> None: - with self.lock: - self._artifacts.pop(run_id, None) - self.router.cache.invalidate() - - # ---- lifecycle - - def create_run(self, **kwargs) -> EvaluationRun: - run = EvaluationRun(**kwargs) - meta = pop_mod.load_meta(run.population_id) - if meta is None: - raise KeyError(f"Unknown population {run.population_id}") - run.partitions_total = meta["num_partitions"] - run.objects_total = meta["num_objects"] - run.status = "queued" - with self.lock: - self.runs[run.run_id] = run - self.save_run(run) - return run - - def start_async(self, run: EvaluationRun) -> None: - t = threading.Thread(target=self._execute, args=(run,), daemon=True) - t.start() - - def execute_sync(self, run: EvaluationRun) -> None: - self._execute(run) - - def _execute(self, run: EvaluationRun) -> None: - try: - run.status = "running" - run.started_at = _now() - t_start = time.perf_counter() - params = model_params(run.model_version, run.overrides) - d = run_dir(run.run_id) - os.makedirs(d, exist_ok=True) - - partials: List[pd.DataFrame] = [] - error_frames: List[pd.DataFrame] = [] - container_frames: List[pd.DataFrame] = [] - conf_hist = QuantileHistogram(0, 1, 64) - iou_hist = QuantileHistogram(0, 1, 64) - hll = HyperLogLog(12) - reservoir = Reservoir(k=200, seed=run.seed) - agg_lock = threading.Lock() - - def work(p: int) -> None: - res = evaluate_partition(run.population_id, p, params, run.seed) - if run.worker_delay_s: - time.sleep(run.worker_delay_s) # simulated distributed I/O latency - # uncompressed npz: these are hot-path worker writes (few MB total) - np.savez(os.path.join(d, f"objects-part-{p:04d}.npz"), **res["objects"]) - np.savez(os.path.join(d, f"fp-part-{p:04d}.npz"), **res["fps"]) - nonlocal conf_hist, iou_hist, hll, reservoir - with agg_lock: - partials.append(res["partial"]) - if len(res["errors"]): - error_frames.append(res["errors"]) - container_frames.append(res["containers"]) - conf_hist = conf_hist.merge(res["conf_hist"]) - iou_hist = iou_hist.merge(res["iou_hist"]) - hll = hll.merge(res["hll"]) - reservoir = reservoir.merge(res["reservoir"]) - run.partitions_done += 1 - run.objects_processed += res["rows"] - elapsed = time.perf_counter() - t_start - run.throughput_objs_per_s = run.objects_processed / max(elapsed, 1e-6) - remaining = run.objects_total - run.objects_processed - run.eta_s = remaining / max(run.throughput_objs_per_s, 1e-6) - - with ThreadPoolExecutor(max_workers=run.workers) as ex: - list(ex.map(work, range(run.partitions_total))) - - run.status = "reducing" - cube_df = cube_mod.reduce_partials(partials) - - run.status = "materializing" - cube_mod.save_cube(os.path.join(d, "cube.npz"), cube_df) - - errors = (pd.concat(error_frames, ignore_index=True) - if error_frames else pd.DataFrame()) - if len(errors): - errors = errors.reset_index(drop=True) - errors["error_id"] = np.arange(len(errors), dtype=np.int64) - risk = (errors["severity"] * 0.5 - + errors["safety_critical"] * 0.3 - + (1 - errors["confidence"]) * 0.15 - + errors["sensor_disagree"] * 0.05) - errors["risk_score"] = np.clip(risk, 0, 1) - np.savez_compressed(os.path.join(d, "errors.npz"), - **{c: errors[c].to_numpy() for c in errors.columns}) - - containers = (pd.concat(container_frames, ignore_index=True) - .groupby(["container_id"] + pop_mod.CONTAINER_DIMS, - as_index=False, observed=True).sum()) - containers["reviewed"] = 0 - containers["verified"] = 0 - err_share = (containers["fn"] + containers["fp"]) / containers["n_objects"].clip(lower=1) - containers["risk_score"] = np.clip( - 0.55 * err_share + 0.25 * (containers["anomalies"] / containers["n_objects"].clip(lower=1)) - + 0.20 * (containers["safety_n"] > 0), 0, 1).round(4) - containers["status"] = np.select( - [containers["risk_score"] >= 0.45, containers["risk_score"] >= 0.2], - [2, 1], default=0).astype(np.int8) - np.savez_compressed(os.path.join(d, "containers.npz"), - **{c: containers[c].to_numpy() for c in containers.columns}) - - self._build_embeddings(d, containers) - - np.savez_compressed(os.path.join(d, "sketches.npz"), - conf_hist=conf_hist.counts, iou_hist=iou_hist.counts, - hll_estimate=np.array(hll.estimate()), - exemplars=np.array(reservoir.items, dtype=np.int64)) - - headline_rows, _ = cube_mod.aggregate( - cube_df, None, None, - ["n", "tp", "fp", "fn", "loc_err", "anomalies", "precision", "recall", "f1", - "mean_iou", "anomaly_rate", "safety_recall", "conf_mean", "error_rate"], 1) - run.headline = headline_rows[0] - run.headline["containers"] = int(containers.shape[0]) - run.headline["containers_hll_estimate"] = round(hll.estimate(), 1) - - per_class_rows, _ = cube_mod.aggregate( - cube_df, None, ["class"], - ["n", "precision", "recall", "f1", "mean_iou", "safety_recall", "anomaly_rate"], 50) - run.per_class = {r["class"]: r for r in per_class_rows} - - run.status = "published" - run.published_at = _now() - run.eta_s = 0.0 - self.save_run(run) - self.invalidate_artifacts(run.run_id) - except Exception as e: # pragma: no cover - defensive - run.status = "failed" - run.error = str(e) - self.save_run(run) - - @staticmethod - def _build_embeddings(d: str, containers: pd.DataFrame) -> None: - """32-d container embeddings: structural features + seeded random projection.""" - feats = [] - nobj = containers["n_objects"].clip(lower=1).to_numpy(dtype=np.float64) - for col in ("tp", "fp", "fn", "anomalies", "safety_n"): - feats.append(containers[col].to_numpy(dtype=np.float64) / nobj) - feats.append(containers["sum_iou"].to_numpy(dtype=np.float64) - / containers["tp"].clip(lower=1).to_numpy(dtype=np.float64)) - feats.append(np.log1p(nobj) / 4.0) - feats.append(containers["risk_score"].to_numpy(dtype=np.float64)) - for dim in pop_mod.CONTAINER_DIMS: - k = len(DIMENSIONS[dim]) - onehot = np.eye(k)[containers[dim].to_numpy(dtype=np.int64)] - feats.extend(onehot.T) - X = np.vstack(feats).T # (n, ~25) - rng = np.random.default_rng(1234) - proj = rng.normal(size=(X.shape[1], 32)) / np.sqrt(X.shape[1]) - emb = X @ proj - emb = emb / np.clip(np.linalg.norm(emb, axis=1, keepdims=True), 1e-9, None) - np.savez_compressed(os.path.join(d, "embeddings.npz"), - container_id=containers["container_id"].to_numpy(), - emb=emb.astype(np.float32)) - - # ---- forensic drill-down (deepest level; loads exactly one partition) - - def container_objects(self, run: EvaluationRun, container_id: int) -> List[Dict]: - p = pop_mod.partition_of_container(container_id) - pop_cols = pop_mod.load_partition(run.population_id, p) - d = run_dir(run.run_id) - with np.load(os.path.join(d, f"objects-part-{p:04d}.npz")) as z: - out_cols = {k: z[k] for k in z.files} - mask = pop_cols["container_id"] == container_id - idx = np.where(mask)[0] - rows = [] - for i in idx.tolist(): - det = bool(out_cols["detected"][i]) - tp = bool(out_cols["tp"][i]) - rows.append({ - "annotation_id": f"obj-{int(pop_cols['object_id'][i])}", - "container_id": int(container_id), - **{k: DIMENSIONS[k][int(pop_cols[k][i])] for k in DIM_NAMES}, - "safety_critical": bool(pop_cols["safety_critical"][i]), - "difficulty": round(float(pop_cols["difficulty"][i]), 4), - "detected": det, - "outcome": "TP" if tp else ("LOCALIZATION" if det else "FN"), - "iou": round(float(out_cols["iou"][i]), 4) if det else None, - "confidence": round(float(out_cols["confidence"][i]), 4) if det else None, - "anomaly": bool(out_cols["anomaly"][i]), - "sensor_disagree": bool(out_cols["sensor_disagree"][i]), - }) - with np.load(os.path.join(d, f"fp-part-{p:04d}.npz")) as z: - fp_cols = {k: z[k] for k in z.files} - fp_mask = fp_cols["container_id"] == container_id - for j in np.where(fp_mask)[0].tolist(): - rows.append({ - "annotation_id": f"fp-{run.run_id[-4:]}-{p}-{j}", - "container_id": int(container_id), - **{k: DIMENSIONS[k][int(fp_cols[k][j])] for k in DIM_NAMES}, - "safety_critical": False, - "difficulty": None, - "detected": True, - "outcome": "FP", - "iou": 0.0, - "confidence": round(float(fp_cols["confidence"][j]), 4), - "anomaly": False, - "sensor_disagree": False, - }) - return rows - - # ---- honest record-scan fallback for the query router - - def scan_records(self, run: EvaluationRun, filters: Optional[Dict], - group_by: Optional[List[str]], metrics: Optional[List[str]], - limit: int) -> tuple: - frames = [] - d = run_dir(run.run_id) - for p in range(run.partitions_total): - pop_cols = pop_mod.load_partition(run.population_id, p) - with np.load(os.path.join(d, f"objects-part-{p:04d}.npz")) as z: - df = pd.DataFrame({ - **{k: pop_cols[k] for k in DIM_NAMES}, - "container_id": pop_cols["container_id"], - "tp": z["tp"].astype(np.int64), - "fn": (1 - z["tp"]).astype(np.int64), - "fp": (z["detected"] & ~z["tp"].astype(bool)).astype(np.int64), - "anomalies": z["anomaly"].astype(np.int64), - "iou": z["iou"], - }) - frames.append(df) - full = pd.concat(frames, ignore_index=True) - full = cube_mod.apply_filters(full, filters) - gb = [g for g in (group_by or []) if g in list(full.columns)] - if not gb: - gb = ["container_id"] - agg = full.groupby(gb, as_index=False, observed=True).agg( - n=("tp", "size"), tp=("tp", "sum"), fp=("fp", "sum"), fn=("fn", "sum"), - anomalies=("anomalies", "sum"), mean_iou=("iou", "mean")) - agg["recall"] = agg["tp"] / (agg["tp"] + agg["fn"]).clip(lower=1) - agg = agg.sort_values("n", ascending=False).head(limit) - rows = [] - for _, r in agg.iterrows(): - row = {} - for g in gb: - row[g] = DIMENSIONS[g][int(r[g])] if g in DIM_NAMES else int(r[g]) - row.update({"n": int(r["n"]), "tp": int(r["tp"]), "fp": int(r["fp"]), - "fn": int(r["fn"]), "anomalies": int(r["anomalies"]), - "mean_iou": round(float(r["mean_iou"]), 6), - "recall": round(float(r["recall"]), 6)}) - rows.append(row) - return rows, len(full) - - -_STORE: Optional[MegaStore] = None -_STORE_LOCK = threading.Lock() - - -def get_mega_store() -> MegaStore: - global _STORE - with _STORE_LOCK: - if _STORE is None: - _STORE = MegaStore() - return _STORE - - -def reset_mega_store() -> None: - global _STORE - with _STORE_LOCK: - _STORE = None diff --git a/hf-space/sensorflow/megaeval/sampling.py b/hf-space/sensorflow/megaeval/sampling.py deleted file mode 100644 index b954743dbf0fa2ede10c09e3cad3d33f8e31448d..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/megaeval/sampling.py +++ /dev/null @@ -1,261 +0,0 @@ -"""Human review as statistical sampling. - -Instead of a flat "review N labels" queue, review effort is a designed sample: - - population -> containers -> suspicious containers -> candidate pool - -> stratified risk-weighted selection -> reviews -> CIs - -Two estimands, each with its own sampling frame: - - precision: frame = predicted positives, strata = confidence tier x safety - - recall: frame = ground-truth objects, strata = model outcome x safety - -Risk weighting happens through *allocation* (risky strata get more samples); -estimates stay honest because strata are combined with their true population -weights (classic stratified estimator): - - p_hat = sum_h W_h * p_hat_h - SE = sqrt( sum_h W_h^2 * p_hat_h (1 - p_hat_h) / n_h ) (normal approx) - -Per-stratum intervals use Wilson. Simulated reviewers agree with ground truth -with 98.5% fidelity (the synthetic stand-in for human accuracy). -""" - -from __future__ import annotations - -import json -import math -import os -from typing import Dict, List, Optional, Tuple - -import numpy as np -import pandas as pd - -from sensorflow.megaeval import population as pop_mod -from sensorflow.megaeval.population import DIM_NAMES -from sensorflow.megaeval.runs import EvaluationRun, MegaStore, run_dir - -HUMAN_FIDELITY = 0.985 -Z95 = 1.959964 - - -def wilson_interval(k: int, n: int, z: float = Z95) -> Tuple[float, float]: - """Wilson score interval for a binomial proportion.""" - if n == 0: - return (0.0, 1.0) - p = k / n - denom = 1 + z**2 / n - center = (p + z**2 / (2 * n)) / denom - margin = z * math.sqrt(p * (1 - p) / n + z**2 / (4 * n**2)) / denom - return (max(0.0, center - margin), min(1.0, center + margin)) - - -def _load_unit_frames(store: MegaStore, run: EvaluationRun) -> Tuple[pd.DataFrame, pd.DataFrame]: - """One batch scan (NOT on the query path) to build the two sampling frames.""" - d = run_dir(run.run_id) - gt_frames, pp_frames = [], [] - for p in range(run.partitions_total): - pop_cols = pop_mod.load_partition(run.population_id, p) - with np.load(os.path.join(d, f"objects-part-{p:04d}.npz")) as z: - detected = z["detected"].astype(bool) - tp = z["tp"].astype(bool) - conf = z["confidence"] - base = pd.DataFrame({ - "unit_id": pop_cols["object_id"], - "container_id": pop_cols["container_id"], - **{k: pop_cols[k] for k in DIM_NAMES}, - "safety": pop_cols["safety_critical"].astype(bool), - "detected": detected, "tp": tp, "confidence": conf, - }) - gt_frames.append(base) - pp = base[detected].copy() - pp["correct"] = pp["tp"] - pp_frames.append(pp) - with np.load(os.path.join(d, f"fp-part-{p:04d}.npz")) as z: - if z["container_id"].size: - fpdf = pd.DataFrame({ - "unit_id": -np.arange(1, z["container_id"].size + 1) - p * 10_000_000, - "container_id": z["container_id"], - **{k: z[k] for k in DIM_NAMES}, - "safety": False, "detected": True, "tp": False, - "confidence": z["confidence"], "correct": False, - }) - pp_frames.append(fpdf) - return pd.concat(gt_frames, ignore_index=True), pd.concat(pp_frames, ignore_index=True) - - -def _stratify_precision(pp: pd.DataFrame) -> pd.Series: - tier = np.select([pp["confidence"] < 0.45, pp["confidence"] < 0.75], [0, 1], default=2) - return pd.Series(tier * 2 + pp["safety"].astype(int).to_numpy(), index=pp.index) - - -def _stratify_recall(gt: pd.DataFrame) -> pd.Series: - return pd.Series((~gt["detected"]).astype(int).to_numpy() * 2 - + gt["safety"].astype(int).to_numpy(), index=gt.index) - - -_PRECISION_STRATA = {0: "low_conf", 1: "low_conf_safety", 2: "mid_conf", 3: "mid_conf_safety", - 4: "high_conf", 5: "high_conf_safety"} -_RECALL_STRATA = {0: "detected", 1: "detected_safety", 2: "missed", 3: "missed_safety"} -# risk-weighted allocation multipliers (riskier strata get proportionally more reviews) -_PRECISION_RISK_W = {0: 3.0, 1: 4.0, 2: 1.5, 3: 2.5, 4: 1.0, 5: 2.0} -_RECALL_RISK_W = {0: 1.0, 1: 2.0, 2: 3.0, 3: 5.0} - - -def _allocate(counts: Dict[int, int], risk_w: Dict[int, float], total_n: int) -> Dict[int, int]: - weights = {h: counts[h] * risk_w.get(h, 1.0) for h in counts} - wsum = sum(weights.values()) or 1.0 - alloc = {} - for h, c in counts.items(): - n_h = int(round(total_n * weights[h] / wsum)) - alloc[h] = max(1, min(n_h, c)) if c > 0 else 0 - return alloc - - -def build_review_plan(store: MegaStore, run: EvaluationRun, - target_n: Optional[int] = None, seed: int = 99) -> Dict: - """Build the sampling funnel + stratified selection. Persists review.json.""" - target_n = target_n or run.lineage["sampling_config"].get("target_n", 1500) - rng = np.random.default_rng(seed + run.seed) - gt, pp = _load_unit_frames(store, run) - art = store.artifacts(run.run_id) - containers = art["containers"] - errors = art.get("errors", pd.DataFrame()) - - suspicious = int((containers["risk_score"] >= 0.2).sum()) - candidates = int(len(errors)) if len(errors) else 0 - - plans = {} - for name, frame, strat_fn, labels, risk_w in ( - ("precision", pp, _stratify_precision, _PRECISION_STRATA, _PRECISION_RISK_W), - ("recall", gt, _stratify_recall, _RECALL_STRATA, _RECALL_RISK_W)): - strata = strat_fn(frame) - counts = strata.value_counts().to_dict() - alloc = _allocate(counts, risk_w, target_n // 2) - selected = {} - for h, n_h in alloc.items(): - if n_h <= 0: - continue - idx = frame.index[strata == h].to_numpy() - chosen = rng.choice(idx, size=min(n_h, idx.size), replace=False) - selected[str(h)] = frame.loc[chosen, "unit_id"].astype(np.int64).tolist() - plans[name] = { - "frame_size": int(len(frame)), - "strata": {str(h): {"label": labels[h], "N": int(counts.get(h, 0)), - "allocated": int(alloc.get(h, 0))} - for h in labels}, - "selected_ids": selected, - } - - selected_total = sum(len(v) for p in plans.values() for v in p["selected_ids"].values()) - review = { - "run_id": run.run_id, - "method": "stratified risk-weighted sampling; Wilson per-stratum, " - "stratified normal-approx combined CI", - "target_n": target_n, - "funnel": { - "population_objects": int(run.objects_total), - "containers": int(len(containers)), - "suspicious_containers": suspicious, - "candidate_pool": candidates, - "statistically_selected": int(selected_total), - "reviewed": 0, - }, - "plans": plans, - "executed": False, - "results": None, - } - with open(os.path.join(run_dir(run.run_id), "review.json"), "w") as f: - json.dump(review, f) - store.invalidate_artifacts(run.run_id) - return review - - -def execute_reviews(store: MegaStore, run: EvaluationRun, seed: int = 7) -> Dict: - """Simulate human review of the selected sample; compute CIs; update the cube.""" - d = run_dir(run.run_id) - with open(os.path.join(d, "review.json")) as f: - review = json.load(f) - rng = np.random.default_rng(seed + run.seed) - gt, pp = _load_unit_frames(store, run) - gt_idx = gt.set_index("unit_id") - pp_idx = pp.set_index("unit_id") - - results = {} - reviewed_units = [] # (dims..., verified) for cube update — recall-frame units only - total_reviewed = 0 - for name, frame_idx, truth_col in (("precision", pp_idx, "correct"), - ("recall", gt_idx, "tp")): - plan = review["plans"][name] - strata_out = [] - p_hat_comb, var_comb = 0.0, 0.0 - frame_N = plan["frame_size"] - n_total = 0 - for h, sinfo in plan["strata"].items(): - ids = plan["selected_ids"].get(h, []) - if not ids: - continue - units = frame_idx.loc[[i for i in ids if i in frame_idx.index]] - if isinstance(units, pd.Series): - units = units.to_frame().T - truth = units[truth_col].astype(bool).to_numpy() - agrees = rng.random(truth.size) < HUMAN_FIDELITY - verdict = np.where(agrees, truth, ~truth) - k, n_h = int(verdict.sum()), int(verdict.size) - W_h = sinfo["N"] / frame_N - p_h = k / n_h if n_h else 0.0 - lo, hi = wilson_interval(k, n_h) - strata_out.append({"stratum": sinfo["label"], "N": sinfo["N"], "n": n_h, - "correct": k, "p": round(p_h, 4), - "wilson_ci": [round(lo, 4), round(hi, 4)], - "weight": round(W_h, 4)}) - p_hat_comb += W_h * p_h - var_comb += (W_h**2) * p_h * (1 - p_h) / max(n_h, 1) - n_total += n_h - total_reviewed += n_h - if name == "recall": - sub = units.copy() - sub["verified"] = verdict - reviewed_units.append(sub) - se = math.sqrt(var_comb) - results[name] = { - "estimate": round(p_hat_comb, 4), - "ci_low": round(max(0.0, p_hat_comb - Z95 * se), 4), - "ci_high": round(min(1.0, p_hat_comb + Z95 * se), 4), - "n_reviewed": n_total, - "frame_size": frame_N, - "method": review["method"], - "strata": strata_out, - } - - # ---- fold reviews back into the cube + container table (reviewed/verified counters) - if reviewed_units: - ru = pd.concat(reviewed_units) - ru = ru.reset_index(drop=True) - upd = ru.groupby(DIM_NAMES, as_index=False, observed=True).agg( - add_reviewed=("verified", "size"), add_verified=("verified", "sum")) - from sensorflow.megaeval import cube as cube_mod - cube_path = os.path.join(d, "cube.npz") - cube_df = cube_mod.load_cube(cube_path) - merged = cube_df.merge(upd, on=DIM_NAMES, how="left") - merged["reviewed"] = merged["reviewed"] + merged["add_reviewed"].fillna(0).astype(np.int64) - merged["verified"] = merged["verified"] + merged["add_verified"].fillna(0).astype(np.int64) - cube_mod.save_cube(cube_path, merged.drop(columns=["add_reviewed", "add_verified"])) - - cupd = ru.groupby("container_id", as_index=False).agg( - add_reviewed=("verified", "size"), add_verified=("verified", "sum")) - cont_path = os.path.join(d, "containers.npz") - with np.load(cont_path) as z: - cont = pd.DataFrame({k: z[k] for k in z.files}) - cont = cont.merge(cupd, on="container_id", how="left") - cont["reviewed"] = cont["reviewed"] + cont["add_reviewed"].fillna(0).astype(np.int64) - cont["verified"] = cont["verified"] + cont["add_verified"].fillna(0).astype(np.int64) - cont = cont.drop(columns=["add_reviewed", "add_verified"]) - np.savez_compressed(cont_path, **{c: cont[c].to_numpy() for c in cont.columns}) - - review["executed"] = True - review["funnel"]["reviewed"] = total_reviewed - review["results"] = results - with open(os.path.join(d, "review.json"), "w") as f: - json.dump(review, f) - store.invalidate_artifacts(run.run_id) - return review diff --git a/hf-space/sensorflow/megaeval/sketches.py b/hf-space/sensorflow/megaeval/sketches.py deleted file mode 100644 index b1476b74ceca224ebd725e426ee617496b3f77bc..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/megaeval/sketches.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Lightweight approximate data structures for population statistics. - -All numpy-based, mergeable (so they can be maintained per partition and reduced), -and each clearly reports whether an answer is exact or approximate. - -- QuantileHistogram: fixed-bin histogram over a known range; percentile queries by - linear interpolation inside the winning bin. Stand-in for a t-digest. -- HyperLogLog: classic HLL with 2^p registers and numpy vectorized updates. -- Reservoir: uniform reservoir sample of exemplar ids (Algorithm R, vectorized-ish). -""" - -from __future__ import annotations - -from typing import Dict, List, Optional - -import numpy as np - -# ---------------------------------------------------------------- hashing - -_SPLITMIX_1 = np.uint64(0x9E3779B97F4A7C15) -_SPLITMIX_2 = np.uint64(0xBF58476D1CE4E5B9) -_SPLITMIX_3 = np.uint64(0x94D049BB133111EB) - - -def _hash64(values: np.ndarray) -> np.ndarray: - """Deterministic 64-bit mix (splitmix64) of integer identifiers.""" - with np.errstate(over="ignore"): - z = values.astype(np.uint64) + _SPLITMIX_1 - z = (z ^ (z >> np.uint64(30))) * _SPLITMIX_2 - z = (z ^ (z >> np.uint64(27))) * _SPLITMIX_3 - return z ^ (z >> np.uint64(31)) - - -# ---------------------------------------------------------------- quantile histogram - - -class QuantileHistogram: - """Fixed-bin histogram over [lo, hi]; approximate percentiles via interpolation. - - Mergeable: histograms with identical bin config add counts element-wise. - """ - - def __init__(self, lo: float = 0.0, hi: float = 1.0, bins: int = 64, - counts: Optional[np.ndarray] = None): - self.lo, self.hi, self.bins = float(lo), float(hi), int(bins) - self.counts = counts if counts is not None else np.zeros(bins, dtype=np.int64) - - def add(self, values: np.ndarray) -> None: - if values.size == 0: - return - clipped = np.clip(values, self.lo, self.hi) - idx = np.minimum( - ((clipped - self.lo) / (self.hi - self.lo) * self.bins).astype(np.int64), - self.bins - 1) - self.counts += np.bincount(idx, minlength=self.bins) - - def merge(self, other: "QuantileHistogram") -> "QuantileHistogram": - assert (self.lo, self.hi, self.bins) == (other.lo, other.hi, other.bins) - return QuantileHistogram(self.lo, self.hi, self.bins, self.counts + other.counts) - - @property - def total(self) -> int: - return int(self.counts.sum()) - - def percentile(self, q: float) -> Optional[float]: - """Approximate q-th percentile (q in [0, 100]).""" - total = self.total - if total == 0: - return None - target = total * (q / 100.0) - cum = np.cumsum(self.counts) - bin_idx = int(np.searchsorted(cum, target, side="left")) - bin_idx = min(bin_idx, self.bins - 1) - prev_cum = cum[bin_idx - 1] if bin_idx > 0 else 0 - in_bin = self.counts[bin_idx] - frac = 0.5 if in_bin == 0 else float(np.clip((target - prev_cum) / in_bin, 0.0, 1.0)) - width = (self.hi - self.lo) / self.bins - return self.lo + (bin_idx + frac) * width - - def to_dict(self) -> Dict: - return {"lo": self.lo, "hi": self.hi, "bins": self.bins, - "counts": self.counts.tolist(), "exact": False} - - @staticmethod - def from_arrays(lo: float, hi: float, counts: np.ndarray) -> "QuantileHistogram": - return QuantileHistogram(lo, hi, len(counts), counts.astype(np.int64)) - - -# ---------------------------------------------------------------- HyperLogLog - - -class HyperLogLog: - """HLL cardinality estimator with 2^p registers (approximate). - - Mergeable: element-wise max of registers. - """ - - def __init__(self, p: int = 12, registers: Optional[np.ndarray] = None): - self.p = int(p) - self.m = 1 << self.p - self.registers = registers if registers is not None else np.zeros(self.m, dtype=np.uint8) - - def add_ids(self, ids: np.ndarray) -> None: - if ids.size == 0: - return - h = _hash64(np.asarray(ids)) - reg_idx = (h >> np.uint64(64 - self.p)).astype(np.int64) - rest = (h << np.uint64(self.p)) | np.uint64((1 << self.p) - 1) - # rank = leading zeros of the remaining bits + 1 - ranks = np.zeros(ids.shape, dtype=np.uint8) - remaining = rest.copy() - # 64-bit leading-zero count via float log2 is unsafe; loop over 64 bits max via vector ops - lz = np.full(ids.shape, 64, dtype=np.int64) - nonzero = remaining != 0 - # bit_length via log2 on uint64 split into hi/lo 32-bit halves (exact for ints < 2^53 per half) - hi = (remaining >> np.uint64(32)).astype(np.float64) - lo = (remaining & np.uint64(0xFFFFFFFF)).astype(np.float64) - bl = np.zeros(ids.shape, dtype=np.int64) - hi_nz = hi > 0 - bl[hi_nz] = 32 + np.floor(np.log2(hi[hi_nz])).astype(np.int64) + 1 - lo_only = (~hi_nz) & (lo > 0) - bl[lo_only] = np.floor(np.log2(lo[lo_only])).astype(np.int64) + 1 - lz[nonzero] = 64 - bl[nonzero] - ranks = np.minimum(lz + 1, 64 - self.p + 1).astype(np.uint8) - np.maximum.at(self.registers, reg_idx, ranks) - - def merge(self, other: "HyperLogLog") -> "HyperLogLog": - assert self.p == other.p - return HyperLogLog(self.p, np.maximum(self.registers, other.registers)) - - def estimate(self) -> float: - m = float(self.m) - alpha = 0.7213 / (1.0 + 1.079 / m) - inv_sum = np.sum(np.exp2(-self.registers.astype(np.float64))) - raw = alpha * m * m / inv_sum - zeros = int(np.sum(self.registers == 0)) - if raw <= 2.5 * m and zeros > 0: # small-range correction - return m * np.log(m / zeros) - return float(raw) - - -# ---------------------------------------------------------------- reservoir sample - - -class Reservoir: - """Uniform reservoir sample of ids (exemplars for drill-down).""" - - def __init__(self, k: int = 200, seed: int = 0): - self.k = int(k) - self.rng = np.random.default_rng(seed) - self.items: List[int] = [] - self.seen = 0 - - def add_ids(self, ids: np.ndarray) -> None: - for v in np.asarray(ids).tolist(): - self.seen += 1 - if len(self.items) < self.k: - self.items.append(int(v)) - else: - j = int(self.rng.integers(0, self.seen)) - if j < self.k: - self.items[j] = int(v) - - def merge(self, other: "Reservoir") -> "Reservoir": - merged = Reservoir(self.k, seed=self.seen + other.seen + 1) - pool = self.items + other.items - merged.seen = self.seen + other.seen - if len(pool) <= self.k: - merged.items = pool - else: - idx = merged.rng.choice(len(pool), size=self.k, replace=False) - merged.items = [pool[i] for i in idx] - return merged diff --git a/hf-space/sensorflow/metrics/__init__.py b/hf-space/sensorflow/metrics/__init__.py deleted file mode 100644 index c0616162918925795c4a8c8d67da51369e2a8f1d..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/metrics/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -from sensorflow.metrics.perception_3d import ( - bev_iou, - compute_map_mar, - mean_orientation_error, - mean_position_error, -) -from sensorflow.metrics.temporal_mot import ( - compute_id_swap_rate, - compute_track_fragmentation_rate, -) -from sensorflow.metrics.resource_profile import compute_resource_profile - -__all__ = [ - "bev_iou", - "compute_map_mar", - "mean_orientation_error", - "mean_position_error", - "compute_id_swap_rate", - "compute_track_fragmentation_rate", - "compute_resource_profile", -] diff --git a/hf-space/sensorflow/metrics/perception_3d.py b/hf-space/sensorflow/metrics/perception_3d.py deleted file mode 100644 index 6336311ec757c8a648d4eaeacdfe988d2b2a24b6..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/metrics/perception_3d.py +++ /dev/null @@ -1,148 +0,0 @@ -"""3D perception metrics: mAP, mAR, IoU, orientation/position error.""" - -from __future__ import annotations - -import math -from typing import Dict, List, Tuple - -import numpy as np - - -def _rotate_corners(cx: float, cy: float, l: float, w: float, yaw: float) -> np.ndarray: - """Return 4 BEV corners of rotated rectangle.""" - cos_y, sin_y = math.cos(yaw), math.sin(yaw) - corners = np.array([ - [-l / 2, -w / 2], [l / 2, -w / 2], [l / 2, w / 2], [-l / 2, w / 2], - ]) - rot = np.array([[cos_y, -sin_y], [sin_y, cos_y]]) - return corners @ rot.T + np.array([cx, cy]) - - -def _polygon_area(poly: np.ndarray) -> float: - x, y = poly[:, 0], poly[:, 1] - return 0.5 * abs(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1))) - - -def _clip_polygon(subject: np.ndarray, clip: np.ndarray) -> np.ndarray: - """Sutherland-Hodgman polygon clipping (convex).""" - def inside(p, edge_start, edge_end): - return (edge_end[0] - edge_start[0]) * (p[1] - edge_start[1]) - \ - (edge_end[1] - edge_start[1]) * (p[0] - edge_start[0]) >= 0 - - def intersection(s, e, cp1, cp2): - dc = cp1 - cp2 - dp = s - e - n1 = cp1[0] * cp2[1] - cp1[1] * cp2[0] - n2 = s[0] * e[1] - s[1] * e[0] - denom = dc[0] * dp[1] - dc[1] * dp[0] - if abs(denom) < 1e-10: - return s - return np.array([(n1 * dp[0] - n2 * dc[0]) / denom, (n1 * dp[1] - n2 * dc[1]) / denom]) - - output = subject.copy() - for i in range(len(clip)): - input_list = output - output = [] - cp1, cp2 = clip[i], clip[(i + 1) % len(clip)] - if len(input_list) == 0: - break - s = input_list[-1] - for e in input_list: - if inside(e, cp1, cp2): - if not inside(s, cp1, cp2): - output.append(intersection(s, e, cp1, cp2)) - output.append(e) - elif inside(s, cp1, cp2): - output.append(intersection(s, e, cp1, cp2)) - s = e - output = np.array(output) if output else np.array([]).reshape(0, 2) - return output - - -def bev_iou(box_a: List[float], box_b: List[float]) -> float: - """BEV rotated rectangle IoU for bbox_3d [x,y,z,l,w,h,yaw].""" - xa, ya, la, wa, yaw_a = box_a[0], box_a[1], box_a[3], box_a[4], box_a[6] - xb, yb, lb, wb, yaw_b = box_b[0], box_b[1], box_b[3], box_b[4], box_b[6] - poly_a = _rotate_corners(xa, ya, la, wa, yaw_a) - poly_b = _rotate_corners(xb, yb, lb, wb, yaw_b) - inter = _clip_polygon(poly_a, poly_b) - if len(inter) < 3: - return 0.0 - inter_area = _polygon_area(inter) - area_a = la * wa - area_b = lb * wb - union = area_a + area_b - inter_area - return inter_area / union if union > 0 else 0.0 - - -def _match_boxes( - preds: List[List[float]], - gts: List[List[float]], - iou_threshold: float, -) -> Tuple[List[Tuple[int, int, float]], List[int], List[int]]: - """Greedy matching by IoU.""" - matched_pairs = [] - matched_gt = set() - matched_pred = set() - ious = [] - for pi, p in enumerate(preds): - for gi, g in enumerate(gts): - iou = bev_iou(p, g) - if iou >= iou_threshold: - ious.append((iou, pi, gi)) - ious.sort(reverse=True) - for iou, pi, gi in ious: - if pi not in matched_pred and gi not in matched_gt: - matched_pairs.append((pi, gi, iou)) - matched_pred.add(pi) - matched_gt.add(gi) - unmatched_pred = [i for i in range(len(preds)) if i not in matched_pred] - unmatched_gt = [i for i in range(len(gts)) if i not in matched_gt] - return matched_pairs, unmatched_pred, unmatched_gt - - -def compute_map_mar( - all_preds: List[List[float]], - all_gts: List[List[float]], - iou_thresholds: List[float] = None, -) -> Dict[str, float]: - """Compute mean AP and mean AR across IoU thresholds.""" - iou_thresholds = iou_thresholds or [0.5, 0.7] - aps, ars = [], [] - for thresh in iou_thresholds: - pairs, unmatched_pred, unmatched_gt = _match_boxes(all_preds, all_gts, thresh) - tp = len(pairs) - fp = len(unmatched_pred) - fn = len(unmatched_gt) - precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0 - recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0 - aps.append(precision) - ars.append(recall) - return { - "map_3d": float(np.mean(aps)) if aps else 0.0, - "mar_3d": float(np.mean(ars)) if ars else 0.0, - "mean_iou_3d": float(np.mean([p[2] for p in _match_boxes(all_preds, all_gts, 0.5)[0]])) if all_preds and all_gts and _match_boxes(all_preds, all_gts, 0.5)[0] else 0.0, - } - - -def mean_orientation_error(matched_pairs: List[Tuple[List[float], List[float]]]) -> float: - """Mean absolute yaw error in degrees.""" - if not matched_pairs: - return 0.0 - errors = [] - for pred, gt in matched_pairs: - diff = abs(pred[6] - gt[6]) - diff = min(diff, 2 * math.pi - diff) - errors.append(math.degrees(diff)) - return float(np.mean(errors)) - - -def mean_position_error(matched_pairs: List[Tuple[List[float], List[float]]]) -> float: - """Mean L2 BEV center distance in meters.""" - if not matched_pairs: - return 0.0 - errors = [] - for pred, gt in matched_pairs: - dist = math.sqrt((pred[0] - gt[0]) ** 2 + (pred[1] - gt[1]) ** 2) - errors.append(dist) - return float(np.mean(errors)) diff --git a/hf-space/sensorflow/metrics/resource_profile.py b/hf-space/sensorflow/metrics/resource_profile.py deleted file mode 100644 index 14390eb6f24d68cea59acdc4facf59a41b4c1d3f..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/metrics/resource_profile.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Resource profiling in deterministic process_units and compute_cycles.""" - -from __future__ import annotations - -from typing import Dict, List - -from sensorflow.schemas.unified_frame import FusedFrame - - -COMPLEXITY_FACTORS = { - "sam": 1.0, - "lift_3d": 0.3, - "tracker": 0.1, - "benchmark": 0.05, -} - - -def compute_resource_profile( - frames: List[FusedFrame], - stages_run: List[str] = None, -) -> Dict[str, int]: - """ - Compute deterministic resource cost without wall-clock time. - process_units = frames * cameras * (lidar_points / 1000) - compute_cycles = process_units * sum(complexity_factors for stages) - """ - stages_run = stages_run or ["sam", "lift_3d", "tracker", "benchmark"] - total_points = sum(f.lidar.num_points for f in frames) - num_cameras = max((len(f.cameras) for f in frames), default=1) - num_frames = len(frames) - - process_units = int(num_frames * num_cameras * max(total_points / max(num_frames, 1) / 1000, 1)) - factor_sum = sum(COMPLEXITY_FACTORS.get(s, 0.1) for s in stages_run) - compute_cycles = int(process_units * factor_sum) - - return { - "process_units": process_units, - "compute_cycles": compute_cycles, - "num_frames": num_frames, - "num_cameras": num_cameras, - "total_lidar_points": total_points, - } diff --git a/hf-space/sensorflow/metrics/temporal_mot.py b/hf-space/sensorflow/metrics/temporal_mot.py deleted file mode 100644 index 7bcc19f373447bddb1611d98fe15551d092bb594..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/metrics/temporal_mot.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Temporal MOT metrics: ID swaps and track fragmentation.""" - -from __future__ import annotations - -from collections import defaultdict -from typing import Dict, List, Tuple - - -def compute_id_swap_rate( - pred_tracks: List[Dict], - gt_tracks: List[Dict], -) -> float: - """ - Count ID swaps: when GT instance_id is stable but pred track_id changes. - pred_tracks/gt_tracks: [{track_id/instance_id, frames: [{frame_id, bbox_3d}]}] - """ - swaps = 0 - total_updates = 0 - - gt_by_instance: Dict[str, Dict[str, List[float]]] = defaultdict(dict) - for gt in gt_tracks: - iid = gt.get("instance_id", gt.get("track_id", "")) - for fr in gt.get("frames", []): - gt_by_instance[iid][fr["frame_id"]] = fr["bbox_3d"] - - pred_by_track: Dict[int, Dict[str, List[float]]] = defaultdict(dict) - for pt in pred_tracks: - tid = pt["track_id"] - for fr in pt.get("frames", []): - pred_by_track[tid][fr["frame_id"]] = fr["bbox_3d"] - - for iid, gt_frames in gt_by_instance.items(): - frame_ids = sorted(gt_frames.keys()) - if len(frame_ids) < 2: - continue - prev_tid = None - for fid in frame_ids: - best_tid = None - best_dist = float("inf") - for tid, pframes in pred_by_track.items(): - if fid in pframes: - dist = _center_dist(pframes[fid], gt_frames[fid]) - if dist < best_dist: - best_dist = dist - best_tid = tid - if best_tid is not None: - total_updates += 1 - if prev_tid is not None and best_tid != prev_tid: - swaps += 1 - prev_tid = best_tid - - return swaps / total_updates if total_updates > 0 else 0.0 - - -def compute_track_fragmentation_rate( - pred_tracks: List[Dict], - gt_tracks: List[Dict], - match_threshold_m: float = 2.0, -) -> float: - """Fraction of GT instances matched by more than one pred track.""" - fragmented = 0 - total_gt = len(gt_tracks) - if total_gt == 0: - return 0.0 - - for gt in gt_tracks: - iid = gt.get("instance_id", gt.get("track_id", "")) - gt_frames = {fr["frame_id"]: fr["bbox_3d"] for fr in gt.get("frames", [])} - matched_tracks = set() - for pt in pred_tracks: - pframes = {fr["frame_id"]: fr["bbox_3d"] for fr in pt.get("frames", [])} - overlap = 0 - for fid, gt_box in gt_frames.items(): - if fid in pframes: - if _center_dist(pframes[fid], gt_box) < match_threshold_m: - overlap += 1 - if overlap >= 2: - matched_tracks.add(pt["track_id"]) - if len(matched_tracks) > 1: - fragmented += 1 - - return fragmented / total_gt - - -def _center_dist(a: List[float], b: List[float]) -> float: - return ((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2) ** 0.5 diff --git a/hf-space/sensorflow/mitl_copilot.py b/hf-space/sensorflow/mitl_copilot.py deleted file mode 100644 index c6bab06ff8e8047d5f4fa8ea0f0abaa96e051209..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/mitl_copilot.py +++ /dev/null @@ -1,136 +0,0 @@ -"""MITL copilot: LLM failure-mode critique and edge-case routing.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, Dict, List, Optional - -import httpx - -OLLAMA_ENDPOINTS = [ - {"url": "http://dgx-spark.tail16d8d9.ts.net:11434/api/chat", "model": "gemma4:26b"}, - {"url": "http://localhost:11434/api/chat", "model": "gemma4:latest"}, -] - -MITL_QUEUE_PATH = Path("runs/pipeline/mitl_queue.json") - - -class MitlCopilot: - """Route edge cases to human review with LLM-generated critiques.""" - - def __init__(self, queue_path: Optional[Path] = None): - self.queue_path = queue_path or MITL_QUEUE_PATH - - def route_edge_cases( - self, - sequence_id: str, - metric_card: Dict[str, float], - pred_tracks: List[Dict], - thresholds: Optional[Dict[str, float]] = None, - ) -> List[Dict]: - thresholds = thresholds or { - "id_swap_rate": 0.02, - "position_error_m": 2.0, - "track_fragmentation_rate": 0.05, - } - queue = self._load_queue() - edge_cases = [] - - if metric_card.get("id_swap_rate", 0) > thresholds["id_swap_rate"]: - edge_cases.append({ - "type": "id_swap", - "severity": "high", - "metric_value": metric_card["id_swap_rate"], - "tracks": pred_tracks[:3], - }) - - if metric_card.get("position_error_m", 0) > thresholds["position_error_m"]: - edge_cases.append({ - "type": "position_error", - "severity": "high", - "metric_value": metric_card["position_error_m"], - "tracks": pred_tracks[:3], - }) - - if metric_card.get("track_fragmentation_rate", 0) > thresholds["track_fragmentation_rate"]: - edge_cases.append({ - "type": "fragmentation", - "severity": "medium", - "metric_value": metric_card["track_fragmentation_rate"], - "tracks": pred_tracks[:3], - }) - - for case in edge_cases: - case["sequence_id"] = sequence_id - case["critique"] = self.generate_critique(case, metric_card) - queue.append(case) - - self._save_queue(queue) - return edge_cases - - def generate_critique( - self, - edge_case: Dict[str, Any], - metric_card: Dict[str, float], - ) -> str: - prompt = f"""You are a Lead QA Auditor for an autonomous driving 3D perception pipeline. -Analyze this failure mode and provide a concise structural critique. - -FAILURE TYPE: {edge_case['type']} -SEVERITY: {edge_case['severity']} -METRIC VALUE: {edge_case['metric_value']} - -FULL METRIC CARD: -{json.dumps(metric_card, indent=2)} - -AFFECTED TRACKS (sample): -{json.dumps(edge_case.get('tracks', [])[:2], indent=2)} - -Provide a professional markdown critique covering: -1. Root cause hypothesis -2. Impact on downstream safety -3. Recommended remediation steps - -Be concise and actionable.""" - - for ep in OLLAMA_ENDPOINTS: - try: - payload = { - "model": ep["model"], - "messages": [{"role": "user", "content": prompt}], - "stream": False, - } - res = httpx.post(ep["url"], json=payload, timeout=25.0) - if res.status_code == 200: - text = res.json().get("message", {}).get("content", "") - if text: - return text - except Exception: - continue - - return self._mock_critique(edge_case) - - def _mock_critique(self, edge_case: Dict) -> str: - return ( - f"### Edge Case: {edge_case['type']}\n\n" - f"Metric value **{edge_case['metric_value']:.4f}** exceeds threshold. " - f"Review affected tracks manually. LLM critique unavailable (Ollama offline)." - ) - - def _load_queue(self) -> List[Dict]: - if self.queue_path.exists(): - with open(self.queue_path) as f: - return json.load(f) - return [] - - def _save_queue(self, queue: List[Dict]): - self.queue_path.parent.mkdir(parents=True, exist_ok=True) - with open(self.queue_path, "w") as f: - json.dump(queue, f, indent=2) - - def get_queue(self, sequence_id: Optional[str] = None) -> List[Dict]: - queue = self._load_queue() - if sequence_id: - return [q for q in queue if q.get("sequence_id") == sequence_id] - return queue diff --git a/hf-space/sensorflow/perception_automator.py b/hf-space/sensorflow/perception_automator.py deleted file mode 100644 index fb2ff2604ad76e9e4eb8fc11167ca11feb8743b4..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/perception_automator.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Automated 2D/3D perception: SAM masks + LiDAR lifting.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import numpy as np - -from sensorflow.schemas.taxonomy_axes import assign_taxonomy_axes -from sensorflow.schemas.unified_frame import FusedFrame, Object3D, UnifiedSequence - - -class PerceptionAutomator: - """SAM-based 2D masks lifted to 3D bounding box proposals.""" - - def __init__( - self, - sam_checkpoint: str = "models/sam_vit_b.pth", - device: str = "cpu", - use_sam: bool = True, - ): - self.sam_checkpoint = sam_checkpoint - self.device = device - self.use_sam = use_sam - self._sam = None - self._mask_generator = None - - def _init_sam(self): - if self._sam is not None: - return - if not self.use_sam: - return - try: - from segment_anything import SamAutomaticMaskGenerator, sam_model_registry - self._sam = sam_model_registry["vit_b"](checkpoint=self.sam_checkpoint) - self._sam.to(self.device) - self._mask_generator = SamAutomaticMaskGenerator(self._sam) - except (ImportError, FileNotFoundError): - self.use_sam = False - - def run_sequence(self, sequence: UnifiedSequence, output_dir: Path) -> Dict[str, List[Object3D]]: - output_dir.mkdir(parents=True, exist_ok=True) - all_proposals: Dict[str, List[Object3D]] = {} - for frame in sequence.frames: - proposals = self.run_frame(frame) - all_proposals[frame.frame_id] = proposals - out_path = output_dir / f"{frame.frame_id}.json" - with open(out_path, "w") as f: - json.dump([p.model_dump() for p in proposals], f, indent=2) - return all_proposals - - def run_frame(self, frame: FusedFrame) -> List[Object3D]: - camera_name = "front" if "front" in frame.cameras else next(iter(frame.cameras), None) - if camera_name is None: - return self._proposals_from_gt(frame) - - camera = frame.cameras[camera_name] - masks = self._sam_masks(camera.image_path) - if not masks: - masks = self._fallback_masks(frame) - points = self._load_lidar(frame.lidar.path) - proposals = self._lift_masks_to_3d(points, masks, frame) - return self._assign_taxonomy(proposals, frame) - - def _sam_masks(self, image_path: str) -> List[Dict]: - self._init_sam() - if not self.use_sam or not self._mask_generator: - return [] - try: - import cv2 - if image_path.startswith("http"): - return [] - img = cv2.imread(image_path) - if img is None: - return [] - img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - return self._mask_generator.generate(img) - except Exception: - return [] - - def _fallback_masks(self, frame: FusedFrame) -> List[Dict]: - """Generate masks from GT 2D annotations or synthetic grid.""" - masks = [] - for i, gt in enumerate(frame.ground_truth): - masks.append({ - "segmentation": None, - "bbox": self._bbox3d_to_2d(gt.bbox_3d), - "area": 1000, - "predicted_iou": gt.confidence, - "stability_score": gt.confidence, - "mask_id": f"mask_{i}", - }) - if not masks: - masks.append({ - "segmentation": None, - "bbox": [100, 100, 200, 200], - "area": 10000, - "predicted_iou": 0.8, - "stability_score": 0.8, - "mask_id": "mask_default", - }) - return masks - - def _bbox3d_to_2d(self, bbox_3d: List[float]) -> List[float]: - x, y = bbox_3d[0], bbox_3d[1] - l, w = bbox_3d[3], bbox_3d[4] - cx = int(320 + x / 0.05) - cy = int(240 - y / 0.05) - bw = int(l / 0.05) - bh = int(w / 0.05) - return [cx - bw // 2, cy - bh // 2, bw, bh] - - def _load_lidar(self, path: Optional[str]) -> np.ndarray: - if path and Path(path).exists(): - pts = np.fromfile(path, dtype=np.float32) - if len(pts) % 3 == 0: - return pts.reshape(-1, 3) - return np.random.randn(500, 3).astype(np.float32) * 2 - - def _lift_masks_to_3d( - self, - points: np.ndarray, - masks: List[Dict], - frame: FusedFrame, - ) -> List[Object3D]: - proposals = [] - for mask in masks: - bbox_2d = mask.get("bbox", [0, 0, 100, 100]) - x1, y1, w, h = bbox_2d - x2, y2 = x1 + w, y1 + h - cx_img = (x1 + x2) / 2 - cy_img = (y1 + y2) / 2 - - x_world = (cx_img - 320) * 0.05 - y_world = (240 - cy_img) * 0.05 - - nearby = points[ - (np.abs(points[:, 0] - x_world) < 3) & - (np.abs(points[:, 1] - y_world) < 3) - ] - if len(nearby) < 3: - nearby = np.array([[x_world, y_world, 0.5]]) - - center = nearby.mean(axis=0) - if len(nearby) >= 3: - cov = np.cov(nearby[:, :2].T) - eigvals, eigvecs = np.linalg.eigh(cov) - yaw = float(np.arctan2(eigvecs[1, -1], eigvecs[0, -1])) - l = float(np.sqrt(max(eigvals[1], 0.1)) * 4) if len(eigvals) > 1 else 2.0 - w = float(np.sqrt(max(eigvals[0], 0.1)) * 4) if len(eigvals) > 0 else 1.0 - else: - yaw = 0.0 - l, w = 2.0, 1.0 - - conf = mask.get("predicted_iou", mask.get("stability_score", 0.5)) - proposals.append(Object3D( - bbox_3d=[float(center[0]), float(center[1]), float(center[2]), - max(l, 0.5), max(w, 0.3), 1.5, yaw], - class_name="unknown", - confidence=float(conf), - sam_mask_ref=mask.get("mask_id", ""), - )) - return proposals - - def _assign_taxonomy(self, proposals: List[Object3D], frame: FusedFrame) -> List[Object3D]: - speed = frame.ego_pose.speed_kmh - for i, prop in enumerate(proposals): - if i < len(frame.ground_truth): - prop.class_name = frame.ground_truth[i].class_name - prop.taxonomy_axes = assign_taxonomy_axes(prop.class_name, speed_kmh=speed) - return proposals - - def _proposals_from_gt(self, frame: FusedFrame) -> List[Object3D]: - return [ - Object3D( - bbox_3d=gt.bbox_3d, - class_name=gt.class_name, - confidence=gt.confidence, - taxonomy_axes=gt.taxonomy_axes, - ) - for gt in frame.ground_truth - ] diff --git a/hf-space/sensorflow/platform/__init__.py b/hf-space/sensorflow/platform/__init__.py deleted file mode 100644 index 08f96f8a8c1f07ac1eb863268cc4af091fd2e493..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/platform/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Population-scale evaluation foundation — extends MegaEval + LabelEval. - -Phase 1: schemas, aggregate levels, metrics, container quality, model compare, -multi-gate skeleton, evidence packages. Phases 2–7 leave TODO hooks only. -""" - -from sensorflow.platform.levels import AggregateLevel, EvaluationScope -from sensorflow.platform.entities import Provenance, EvidencePackage - -__all__ = [ - "AggregateLevel", - "EvaluationScope", - "Provenance", - "EvidencePackage", -] diff --git a/hf-space/sensorflow/platform/api.py b/hf-space/sensorflow/platform/api.py deleted file mode 100644 index 6c0cbd24cc58e184c776fa82d95ad1fda232f5cd..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/platform/api.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Platform Phase 1 FastAPI routes — mount into existing app_backend. - -Routes: - /api/evaluations/... - /api/containers/... - /api/models/compare - /api/gates/... -""" - -from __future__ import annotations - -from typing import Any, Dict, List, Optional - -from fastapi import APIRouter, HTTPException -from pydantic import BaseModel, Field - -from sensorflow.platform import levels -from sensorflow.platform.container_quality import build_container_quality_profile -from sensorflow.platform.compare import compare_models -from sensorflow.platform.evidence import build_evidence_package, export_evidence_package -from sensorflow.platform.gates import evaluate_multi_gates, list_gate_defs, load_gate_config, save_gate_config -from sensorflow.platform.levels import AggregateLevel, EvaluationScope - -router = APIRouter(prefix="/api") - - -# ------------------------------------------------------------------ requests - - -class EvaluationCreateRequest(BaseModel): - level: AggregateLevel = AggregateLevel.POPULATION - dataset_id: Optional[str] = None - population_id: Optional[str] = None - run_id: Optional[str] = None - container_id: Optional[str] = None - mega_container_id: Optional[int] = None - drive_id: Optional[str] = None - scene_id: Optional[str] = None - sequence_id: Optional[str] = None - frame_id: Optional[str] = None - cohort_filters: Dict[str, List[str]] = Field(default_factory=dict) - model_version: Optional[str] = None - label_version: Optional[str] = None - - -class ModelCompareRequest(BaseModel): - run_ids: List[str] = Field(..., min_length=2) - baseline_run_id: Optional[str] = None - policy: Dict[str, Any] = Field(default_factory=dict) - - -class GateEvaluateRequest(BaseModel): - sequence_id: Optional[str] = None - candidate_run_id: Optional[str] = None - baseline_run_id: Optional[str] = None - - -class GateConfigUpdate(BaseModel): - gates: Dict[str, Any] = Field(default_factory=dict) - - -class EvidenceExportRequest(BaseModel): - evaluation_id: Optional[str] = None - run_id: Optional[str] = None - dataset_id: Optional[str] = None - sequence_id: Optional[str] = None - candidate_run_id: Optional[str] = None - baseline_run_id: Optional[str] = None - persist: bool = True - - -# ------------------------------------------------------------------ evaluations - - -@router.get("/evaluations/levels") -def evaluation_levels(): - return { - "levels": [lvl.value for lvl in AggregateLevel.ladder()], - "backends": {lvl.value: levels.LEVEL_BACKEND[lvl] for lvl in AggregateLevel}, - } - - -@router.post("/evaluations/scope") -def create_evaluation_scope(req: EvaluationCreateRequest): - scope = EvaluationScope(**req.model_dump()) - return { - "scope": levels.summarize_scope(scope), - "message": "Phase 1 scope descriptor — queries still served by MegaEval/LabelEval backends", - } - - -@router.get("/evaluations/{run_id}") -def get_evaluation_summary(run_id: str): - """Unified evaluation summary for a MegaEval run (async job status preserved).""" - try: - from sensorflow.megaeval.runs import get_mega_store - run = get_mega_store().runs.get(run_id) - except Exception as exc: - raise HTTPException(500, str(exc)) from exc - if run is None: - raise HTTPException(404, f"Unknown evaluation run {run_id}") - scope = EvaluationScope( - level=AggregateLevel.POPULATION, - evaluation_id=run.run_id, - run_id=run.run_id, - population_id=run.population_id, - model_version=run.model_version, - label_version=getattr(run, "label_version", None), - ) - return { - "evaluation_id": run.run_id, - "status": run.status, - "scope": levels.summarize_scope(scope), - "headline": run.headline, - "model_version": run.model_version, - "population_id": run.population_id, - "lineage": getattr(run, "lineage", None), - "progress": getattr(run, "progress", None), - } - - -@router.post("/evaluations/evidence") -def export_evidence(req: EvidenceExportRequest): - package = build_evidence_package( - evaluation_id=req.evaluation_id, - run_id=req.run_id, - dataset_id=req.dataset_id, - sequence_id=req.sequence_id, - candidate_run_id=req.candidate_run_id, - baseline_run_id=req.baseline_run_id, - ) - path = None - if req.persist: - path = str(export_evidence_package(package)) - return {"package": package.model_dump(), "path": path} - - -# ------------------------------------------------------------------ containers - - -@router.get("/containers/quality") -def containers_quality( - run_id: str, - sort: str = "highest_risk", - limit: int = 50, - offset: int = 0, -): - try: - return build_container_quality_profile( - run_id, sort=sort, limit=limit, offset=offset - ) - except KeyError as exc: - raise HTTPException(404, str(exc)) from exc - except RuntimeError as exc: - raise HTTPException(409, str(exc)) from exc - - -@router.get("/containers/{container_id}/quality") -def container_quality(container_id: int, run_id: str): - try: - return build_container_quality_profile(run_id, container_id=container_id) - except KeyError as exc: - raise HTTPException(404, str(exc)) from exc - except RuntimeError as exc: - raise HTTPException(409, str(exc)) from exc - - -# ------------------------------------------------------------------ models compare - - -@router.post("/models/compare") -def models_compare(req: ModelCompareRequest): - try: - return compare_models( - req.run_ids, - baseline_run_id=req.baseline_run_id, - policy=req.policy or None, - ) - except ValueError as exc: - raise HTTPException(400, str(exc)) from exc - except KeyError as exc: - raise HTTPException(404, str(exc)) from exc - except RuntimeError as exc: - raise HTTPException(409, str(exc)) from exc - - -# ------------------------------------------------------------------ gates - - -@router.get("/gates") -def gates_list(): - cfg = load_gate_config() - return { - "config_path": str(cfg.get("schema_version")), - "gates": [g.model_dump() for g in list_gate_defs(cfg)], - "order": ["scenario", "coverage", "regression", "safety", "quality", "launch", "release"], - } - - -@router.get("/gates/config") -def gates_config(): - return load_gate_config() - - -@router.post("/gates/config") -def gates_config_update(req: GateConfigUpdate): - cfg = load_gate_config() - for name, patch in req.gates.items(): - if name not in cfg["gates"]: - cfg["gates"][name] = patch - else: - cfg["gates"][name].update(patch) - save_gate_config(cfg) - return cfg - - -@router.post("/gates/evaluate") -def gates_evaluate(req: GateEvaluateRequest): - return evaluate_multi_gates( - sequence_id=req.sequence_id, - candidate_run_id=req.candidate_run_id, - baseline_run_id=req.baseline_run_id, - ) - - -@router.get("/gates/status") -def gates_status( - sequence_id: Optional[str] = None, - candidate_run_id: Optional[str] = None, - baseline_run_id: Optional[str] = None, -): - """Multi-gate readiness skeleton for UI cards.""" - return evaluate_multi_gates( - sequence_id=sequence_id, - candidate_run_id=candidate_run_id, - baseline_run_id=baseline_run_id, - ) diff --git a/hf-space/sensorflow/platform/compare.py b/hf-space/sensorflow/platform/compare.py deleted file mode 100644 index ee2212b54991bba747741e9e1575268b5ec8c26a..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/platform/compare.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Multi-model comparison (A vs B vs C) wrapping MegaEval comparative analytics. - -Does not invent a second compare engine — delegates pairwise compare_runs and -folds results into a multi-way delta table. -""" - -from __future__ import annotations - -from typing import Any, Dict, List, Optional - -from sensorflow.megaeval import analysis -from sensorflow.megaeval.runs import MegaStore, get_mega_store -from sensorflow.platform.metrics_engine import delta_metrics, round4 - - -def _headline_map(run) -> Dict[str, Optional[float]]: - h = run.headline or {} - return {k: h.get(k) for k in ( - "precision", "recall", "f1", "mean_iou", "safety_recall", "anomaly_rate" - )} - - -def compare_models( - run_ids: List[str], - *, - baseline_run_id: Optional[str] = None, - policy: Optional[Dict] = None, - store: Optional[MegaStore] = None, -) -> Dict[str, Any]: - """Compare 2+ published MegaEval runs (model versions A/B/C…). - - If baseline_run_id is omitted, the first id is the baseline. - """ - if len(run_ids) < 2: - raise ValueError("compare_models requires at least two run_ids") - store = store or get_mega_store() - baseline_id = baseline_run_id or run_ids[0] - if baseline_id not in run_ids: - run_ids = [baseline_id, *run_ids] - - runs = [] - for rid in run_ids: - run = store.runs.get(rid) - if run is None: - raise KeyError(f"Unknown evaluation run {rid}") - if run.status != "published": - raise RuntimeError(f"Run {rid} is {run.status}; results require published") - runs.append(run) - - baseline = next(r for r in runs if r.run_id == baseline_id) - candidates = [r for r in runs if r.run_id != baseline_id] - - models = [{ - "run_id": r.run_id, - "model_version": r.model_version, - "role": "baseline" if r.run_id == baseline_id else "candidate", - "headline": _headline_map(r), - } for r in runs] - - # Multi-way metric table: one row per metric, columns per model - metric_keys = ["precision", "recall", "f1", "mean_iou", "safety_recall", "anomaly_rate"] - matrix = [] - for key in metric_keys: - row: Dict[str, Any] = {"metric": key} - for r in runs: - row[r.model_version] = round4(r.headline.get(key)) if r.headline.get(key) is not None else None - base_v = baseline.headline.get(key) - deltas = {} - for c in candidates: - cv = c.headline.get(key) - if base_v is not None and cv is not None: - deltas[c.model_version] = round4(float(cv) - float(base_v)) - row["deltas_vs_baseline"] = deltas - matrix.append(row) - - pairwise = [] - recommendations = [] - for cand in candidates: - cmp = analysis.compare_runs(store, cand, baseline, policy) - pairwise.append({ - "baseline_run_id": baseline.run_id, - "candidate_run_id": cand.run_id, - "baseline_model": baseline.model_version, - "candidate_model": cand.model_version, - "recommendation": cmp.get("recommendation"), - "blockers": cmp.get("blockers") or [], - "headline": cmp.get("headline_deltas") or delta_metrics( - _headline_map(baseline), _headline_map(cand) - ), - "per_class": cmp.get("per_class") or [], - "regressions": (cmp.get("regressions") or cmp.get("worst_cohorts") or [])[:10], - }) - recommendations.append({ - "candidate": cand.model_version, - "recommendation": cmp.get("recommendation"), - "blocker_count": len(cmp.get("blockers") or []), - }) - - # Also surface LabelEval registry models when present (non-fatal) - registry = [] - try: - from sensorflow.evaluation.records import get_store - for m in get_store().all("models"): - registry.append({ - "model_id": m.model_id, - "model_version": m.model_version, - "metrics": m.metrics.model_dump() if hasattr(m.metrics, "model_dump") else {}, - "regression_status": m.regression_status, - }) - except Exception: - pass - - return { - "baseline_run_id": baseline.run_id, - "run_ids": run_ids, - "models": models, - "metric_matrix": matrix, - "pairwise": pairwise, - "recommendations": recommendations, - "label_eval_registry": registry, - "policy": {**analysis.DEFAULT_PROMOTION_POLICY, **(policy or {})}, - } diff --git a/hf-space/sensorflow/platform/container_quality.py b/hf-space/sensorflow/platform/container_quality.py deleted file mode 100644 index 92268863423a6e2ee6fd03e26eae392d98e8e06b..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/platform/container_quality.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Container quality profiles — wraps MegaEval container table + metric engine.""" - -from __future__ import annotations - -from typing import Any, Dict, List, Optional - -import numpy as np - -from sensorflow.megaeval import population as pop_mod -from sensorflow.megaeval.population import DIMENSIONS -from sensorflow.megaeval.runs import CONTAINER_STATUS, EvaluationRun, MegaStore, get_mega_store -from sensorflow.platform.metrics_engine import container_quality_metrics, aggregate_container_rows - - -def _enrich_row(r: Dict[str, Any]) -> Dict[str, Any]: - tp, fp, fn = int(r["tp"]), int(r["fp"]), int(r["fn"]) - sum_iou = float(r.get("sum_iou") or 0.0) - n = int(r.get("n_objects") or 0) - verified = int(r.get("verified") or 0) - reviewed = int(r.get("reviewed") or 0) - # MegaEval does not yet track disputed/auto_accept/hitl separately; - # approximate: reviewed-verified ≈ HITL queue residue; rest unverified. - hitl = max(0, reviewed - verified) - auto_accepted = 0 # TODO(Phase 2): wire triage auto-accept - disputed = 0 # TODO(Phase 2): grader disagreement → disputed - metrics = container_quality_metrics( - tp=tp, fp=fp, fn=fn, sum_iou=sum_iou, n_objects=n, - verified=verified, reviewed=reviewed, disputed=disputed, - auto_accepted=auto_accepted, hitl=hitl, - anomalies=int(r.get("anomalies") or 0), - ) - dims = {dim: DIMENSIONS[dim][int(r[dim])] for dim in pop_mod.CONTAINER_DIMS if dim in r} - return { - "container_id": int(r["container_id"]), - "status": CONTAINER_STATUS[int(r["status"])] if "status" in r else None, - "risk_score": round(float(r.get("risk_score") or 0), 4), - "dims": dims, - "drill_down": { - "object_endpoint": f"/api/megaeval/runs/{{run_id}}/containers/{int(r['container_id'])}/objects", - "annotation_ids": [], # forensic load on demand - }, - **metrics, - } - - -def build_container_quality_profile( - run_id: str, - *, - container_id: Optional[int] = None, - sort: str = "highest_risk", - limit: int = 50, - offset: int = 0, - store: Optional[MegaStore] = None, -) -> Dict[str, Any]: - store = store or get_mega_store() - run = store.runs.get(run_id) - if run is None: - raise KeyError(f"Unknown evaluation run {run_id}") - if run.status != "published": - raise RuntimeError(f"Run {run_id} is {run.status}; publish before quality profile") - - art = store.artifacts(run_id) - df = art["containers"].copy() - tp, fp, fn = df["tp"], df["fp"], df["fn"] - with np.errstate(divide="ignore", invalid="ignore"): - df["recall"] = np.where((tp + fn) > 0, tp / (tp + fn), np.nan) - df["precision"] = np.where((tp + fp) > 0, tp / (tp + fp), np.nan) - df["mean_iou"] = np.where(tp > 0, df["sum_iou"] / tp, np.nan) - - if container_id is not None: - df = df[df["container_id"] == int(container_id)] - if df.empty: - raise KeyError(f"Unknown container {container_id} in run {run_id}") - - sort_map = { - "highest_risk": ("risk_score", False), - "worst_recall": ("recall", True), - "worst_precision": ("precision", True), - "worst_iou": ("mean_iou", True), - "most_anomalies": ("anomalies", False), - "least_verified": ("verified", True), - } - metric, ascending = sort_map.get(sort, sort_map["highest_risk"]) - df = df.sort_values(metric, ascending=ascending, na_position="last") - total = len(df) - page = df.iloc[offset: offset + max(1, min(limit, 500))] - - rows: List[Dict[str, Any]] = [] - for _, series in page.iterrows(): - rows.append(_enrich_row(series.to_dict())) - - summary = aggregate_container_rows(rows) if rows else container_quality_metrics( - tp=0, fp=0, fn=0 - ) - - return { - "run_id": run_id, - "model_version": run.model_version, - "population_id": run.population_id, - "container_id": container_id, - "total": total, - "sort": sort, - "offset": offset, - "limit": limit, - "summary": summary, - "containers": rows, - "verification_headline": summary.get("verification"), - } - - -def profile_from_counts( - *, - container_id: str, - tp: int, - fp: int, - fn: int, - sum_iou: float = 0.0, - n_objects: int = 0, - verified: int = 0, - reviewed: int = 0, - disputed: int = 0, - auto_accepted: int = 0, - hitl: int = 0, - anomalies: int = 0, -) -> Dict[str, Any]: - """Unit-testable path without MegaStore.""" - metrics = container_quality_metrics( - tp=tp, fp=fp, fn=fn, sum_iou=sum_iou, n_objects=n_objects or (tp + fp), - verified=verified, reviewed=reviewed, disputed=disputed, - auto_accepted=auto_accepted, hitl=hitl, anomalies=anomalies, - ) - return {"container_id": container_id, **metrics} diff --git a/hf-space/sensorflow/platform/entities.py b/hf-space/sensorflow/platform/entities.py deleted file mode 100644 index ac96ec713017c5248a344143e7d39cbc0b84e4f1..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/platform/entities.py +++ /dev/null @@ -1,306 +0,0 @@ -"""Platform entity schemas with versioning / provenance. - -Prefer these as the shared vocabulary across LabelEval + MegaEval. -Existing store models in evaluation.records remain the persistence layer for -label QA; MegaEval continues to use npz cubes. These Pydantic models are the -contract for APIs, evidence packages, and later lakehouse rows. -""" - -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Any, Dict, List, Literal, Optional -from uuid import uuid4 - -from pydantic import BaseModel, Field - - -def _now() -> str: - return datetime.now(timezone.utc).isoformat() - - -def _id(prefix: str) -> str: - return f"{prefix}-{uuid4().hex[:10]}" - - -class Provenance(BaseModel): - """Lineage fields required on durable evaluation artifacts.""" - - created_at: str = Field(default_factory=_now) - created_by: str = "system" - schema_version: str = "platform-1.0" - parent_ids: List[str] = Field(default_factory=list) - source_system: str = "sensorflow" # labeleval | megaeval | pipeline | platform - seed: Optional[int] = None - code_version: str = "phase1" - notes: str = "" - - -class VersionedEntity(BaseModel): - version: str = "v1" - provenance: Provenance = Field(default_factory=Provenance) - - -# ------------------------------------------------------------------ hierarchy - - -class DatasetEntity(VersionedEntity): - dataset_id: str - name: str = "" - population_id: Optional[str] = None # MegaEval link - num_frames: int = 0 - gt_type: Optional[str] = None - - -class Container(VersionedEntity): - """Scene/segment holding related objects (MegaEval container_id).""" - - container_id: str - dataset_id: Optional[str] = None - run_id: Optional[str] = None - mega_container_id: Optional[int] = None - drive_id: Optional[str] = None - scene_id: Optional[str] = None - dims: Dict[str, str] = Field(default_factory=dict) - n_objects: int = 0 - - -class Group(VersionedEntity): - """Logical grouping: verification bucket OR cohort alias.""" - - group_id: str - kind: Literal["verification", "cohort", "custom"] = "custom" - dataset_id: Optional[str] = None - member_ids: List[str] = Field(default_factory=list) - label: str = "" - - -class Drive(VersionedEntity): - drive_id: str - dataset_id: Optional[str] = None - scene_ids: List[str] = Field(default_factory=list) - container_ids: List[str] = Field(default_factory=list) - description: str = "" - - -class SceneEntity(VersionedEntity): - scene_id: str - dataset_id: Optional[str] = None - drive_id: Optional[str] = None - sequence_ids: List[str] = Field(default_factory=list) - description: str = "" - - -class ScenarioEntity(VersionedEntity): - scenario_id: str - scenario_type: str - dataset_id: Optional[str] = None - frame_ids: List[str] = Field(default_factory=list) - description: str = "" - - -class FrameEntity(VersionedEntity): - frame_id: str - dataset_id: Optional[str] = None - scene_id: Optional[str] = None - sequence_id: Optional[str] = None - container_id: Optional[str] = None - index: int = 0 - timestamp_us: int = 0 - - -class Sensor(VersionedEntity): - sensor_id: str - modality: Literal["lidar", "camera", "radar", "ultrasonic", "other"] = "lidar" - frame_id: Optional[str] = None - calibration_ref: Optional[str] = None - - -class ObjectEntity(VersionedEntity): - object_id: str - class_name: str = "vehicle" - frame_id: Optional[str] = None - container_id: Optional[str] = None - track_id: Optional[str] = None - bbox_3d: Optional[List[float]] = None - - -class Trajectory(VersionedEntity): - trajectory_id: str - track_id: Optional[str] = None - object_ids: List[str] = Field(default_factory=list) - frame_ids: List[str] = Field(default_factory=list) - - -class Label(VersionedEntity): - """Alias contract for Annotation — hypothesis, never assumed correct.""" - - label_id: str - annotation_id: Optional[str] = None - frame_id: Optional[str] = None - object_id: Optional[str] = None - class_name: str = "" - confidence: float = 0.0 - status: str = "PENDING" - model_version: Optional[str] = None - - -class AnnotationEntity(Label): - """Explicit Annotation mirror for API clarity.""" - - pass - - -# ------------------------------------------------------------------ models / eval - - -class ModelEntity(VersionedEntity): - model_id: str - name: str = "" - status: str = "ready" - - -class ModelVersion(VersionedEntity): - model_version_id: str - model_id: str - model_version: str - checkpoint_ref: Optional[str] = None - trained_on_dataset: Optional[str] = None - metrics: Dict[str, Any] = Field(default_factory=dict) - - -class Metric(VersionedEntity): - metric_id: str - name: str - value: Optional[float] = None - unit: str = "ratio" - higher_is_better: bool = True - support: Optional[int] = None - level: str = "container" # AggregateLevel value - exact: bool = True - - -class Evaluation(VersionedEntity): - evaluation_id: str - level: str - dataset_id: Optional[str] = None - population_id: Optional[str] = None - run_id: Optional[str] = None - model_version: Optional[str] = None - label_version: Optional[str] = None - scope_ref: Optional[str] = None # container/cohort/drive id - metrics: List[Metric] = Field(default_factory=list) - status: str = "created" - - -class Cohort(VersionedEntity): - cohort_id: str - filters: Dict[str, List[str]] = Field(default_factory=dict) - group_by: List[str] = Field(default_factory=list) - n: int = 0 - run_id: Optional[str] = None - - -class Embedding(VersionedEntity): - embedding_id: str - ref_type: str = "container" # container | object | frame - ref_id: str - dim: int = 0 - # TODO(Phase 3): vector payload / external vector-DB handle - vector_ref: Optional[str] = None - - -class Anomaly(VersionedEntity): - anomaly_id: str - annotation_id: Optional[str] = None - container_id: Optional[str] = None - score: float = 0.0 - is_anomaly: bool = False - detector_scores: Dict[str, float] = Field(default_factory=dict) - - -class Conflict(VersionedEntity): - """Safety / label conflict placeholder — Phase 4 fills SSAM DRAC/DeltaS.""" - - conflict_id: str - kind: str = "label" # label | trajectory | ssam_drac | ssam_deltas - entity_ids: List[str] = Field(default_factory=list) - severity: str = "medium" - # TODO(Phase 4): full SSAM conflict engine fields - detail: Dict[str, Any] = Field(default_factory=dict) - - -class ODDDefinition(VersionedEntity): - odd_id: str - name: str - axes: Dict[str, List[str]] = Field(default_factory=dict) - # TODO(Phase 5): combinatorial coverage rules - description: str = "" - - -class ODDObservation(VersionedEntity): - observation_id: str - odd_id: str - observed_axes: Dict[str, str] = Field(default_factory=dict) - covered: bool = False - support: int = 0 - run_id: Optional[str] = None - - -class QualityGateDef(VersionedEntity): - gate_id: str - name: str - gate_type: Literal[ - "scenario", "coverage", "regression", "safety", "release", "quality", "launch" - ] - thresholds: Dict[str, float] = Field(default_factory=dict) - config_path: Optional[str] = None - enabled: bool = True - - -class GateResult(VersionedEntity): - result_id: str - gate_id: str - gate_type: str - passed: Optional[bool] = None # None = not yet evaluated (skeleton) - failures: List[Dict[str, Any]] = Field(default_factory=list) - metrics: Dict[str, Any] = Field(default_factory=dict) - thresholds: Dict[str, float] = Field(default_factory=dict) - scope_ref: Optional[str] = None - ready: bool = False # False until Phase N wires real checks - message: str = "" - - -class HumanReviewEntity(VersionedEntity): - review_id: str - task_id: Optional[str] = None - annotation_id: Optional[str] = None - reviewer: str = "human-reviewer" - action: Dict[str, Any] = Field(default_factory=dict) - - -class ComputeUsage(VersionedEntity): - usage_id: str = Field(default_factory=lambda: _id("compute")) - stage: str = "" - units: int = 0 - run_id: str = "" - evaluation_id: Optional[str] = None - hardware: str = "cpu" - - -class EvidencePackage(VersionedEntity): - """Exportable evaluation artifact — Phase 1 stub with real fields when available.""" - - package_id: str = Field(default_factory=lambda: _id("evidence")) - evaluation_id: str - dataset_id: Optional[str] = None - dataset_version: Optional[str] = None - population_id: Optional[str] = None - model_versions: List[str] = Field(default_factory=list) - label_version: Optional[str] = None - metrics: Dict[str, Any] = Field(default_factory=dict) - thresholds: Dict[str, Any] = Field(default_factory=dict) - gate_results: List[Dict[str, Any]] = Field(default_factory=list) - compute_usage: List[Dict[str, Any]] = Field(default_factory=list) - drill_down_ids: Dict[str, List[str]] = Field(default_factory=dict) - placeholders: List[str] = Field(default_factory=list) diff --git a/hf-space/sensorflow/platform/evidence.py b/hf-space/sensorflow/platform/evidence.py deleted file mode 100644 index ea7d156f0b7ce78b6f972b34ee569dd2a7029ac6..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/platform/evidence.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Evidence package export — JSON stub with real fields when available.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, Dict, List, Optional - -from sensorflow.platform.entities import EvidencePackage, Provenance, _id - -EVIDENCE_DIR = Path("runs/platform/evidence") - - -def build_evidence_package( - *, - evaluation_id: Optional[str] = None, - run_id: Optional[str] = None, - dataset_id: Optional[str] = None, - sequence_id: Optional[str] = None, - candidate_run_id: Optional[str] = None, - baseline_run_id: Optional[str] = None, - include_gates: bool = True, -) -> EvidencePackage: - placeholders: List[str] = [] - metrics: Dict[str, Any] = {} - model_versions: List[str] = [] - population_id = None - label_version = None - dataset_version = None - compute_usage: List[Dict[str, Any]] = [] - drill: Dict[str, List[str]] = {} - thresholds: Dict[str, Any] = {} - gate_results: List[Dict[str, Any]] = [] - - eval_id = evaluation_id or run_id or _id("eval") - - # MegaEval run headline - if run_id or candidate_run_id: - rid = run_id or candidate_run_id - try: - from sensorflow.megaeval.runs import get_mega_store - store = get_mega_store() - run = store.runs.get(rid) # type: ignore[arg-type] - if run: - metrics["headline"] = dict(run.headline or {}) - model_versions.append(run.model_version) - population_id = run.population_id - label_version = getattr(run, "label_version", None) - drill["run_ids"] = [run.run_id] - else: - placeholders.append(f"mega_run_missing:{rid}") - except Exception as exc: - placeholders.append(f"mega_run_error:{exc}") - else: - placeholders.append("run_id") - - if baseline_run_id: - try: - from sensorflow.megaeval.runs import get_mega_store - br = get_mega_store().runs.get(baseline_run_id) - if br: - model_versions.append(br.model_version) - drill.setdefault("run_ids", []).append(br.run_id) - except Exception: - placeholders.append("baseline_run") - - # LabelEval dataset + process units - if dataset_id: - try: - from sensorflow.evaluation.records import get_store - from sensorflow.evaluation.process_units import usage_summary - store = get_store() - ds = store.get("datasets", dataset_id) - if ds: - dataset_version = ds.version - drill["dataset_ids"] = [dataset_id] - else: - placeholders.append(f"dataset_missing:{dataset_id}") - summary = usage_summary(store) - compute_usage = [summary] if isinstance(summary, dict) else list(summary or []) - except Exception as exc: - placeholders.append(f"labeleval_error:{exc}") - compute_usage = [] - else: - placeholders.append("dataset_id") - - if include_gates: - try: - from sensorflow.platform.gates import evaluate_multi_gates, load_gate_config - cfg = load_gate_config() - thresholds = {k: v.get("thresholds", {}) for k, v in cfg.get("gates", {}).items()} - gate_payload = evaluate_multi_gates( - sequence_id=sequence_id, - candidate_run_id=candidate_run_id or run_id, - baseline_run_id=baseline_run_id, - ) - gate_results = gate_payload.get("gates") or [] - except Exception as exc: - placeholders.append(f"gates_error:{exc}") - - if not metrics: - placeholders.append("metrics") - if not model_versions: - placeholders.append("model_versions") - model_versions = ["placeholder-model"] - if not compute_usage: - compute_usage = [{"stage": "platform_evidence", "units": 0, "note": "placeholder"}] - placeholders.append("compute_usage_detail") - - return EvidencePackage( - package_id=_id("evidence"), - evaluation_id=eval_id, - dataset_id=dataset_id, - dataset_version=dataset_version, - population_id=population_id, - model_versions=model_versions, - label_version=label_version, - metrics=metrics, - thresholds=thresholds, - gate_results=gate_results, - compute_usage=compute_usage, - drill_down_ids=drill, - placeholders=sorted(set(placeholders)), - provenance=Provenance( - source_system="platform", - notes="Phase 1 evidence stub — fill placeholders in later phases", - parent_ids=[x for x in [run_id, dataset_id, sequence_id] if x], - ), - ) - - -def export_evidence_package(package: EvidencePackage, path: Optional[Path] = None) -> Path: - EVIDENCE_DIR.mkdir(parents=True, exist_ok=True) - path = path or (EVIDENCE_DIR / f"{package.package_id}.json") - # Append-only: never overwrite an existing package id silently - if path.exists(): - path = EVIDENCE_DIR / f"{package.package_id}-{_id('rev')}.json" - with open(path, "w") as f: - json.dump(package.model_dump(), f, indent=2) - return path diff --git a/hf-space/sensorflow/platform/gates.py b/hf-space/sensorflow/platform/gates.py deleted file mode 100644 index 76d8d98d1b52939df13a25474f981145b90622e6..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/platform/gates.py +++ /dev/null @@ -1,264 +0,0 @@ -"""Multi-gate readiness skeleton — config-driven thresholds. - -Extends existing QualityGate + LaunchGateEvaluator toward: - Scenario | Coverage | Regression | Safety | Release - -Phases 4–6 fill Safety/ODD/Release deeply; Phase 1 returns honest ready=False -for unwired gates and real results for quality/launch when sequence artifacts exist. -""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, Dict, List, Optional - -from sensorflow.platform.entities import GateResult, QualityGateDef, Provenance, _id - -DEFAULT_GATE_CONFIG_PATH = Path("runs/platform/gate_config.json") - -# Config-driven defaults — never hard-code safety cutoffs in callers. -DEFAULT_GATE_CONFIG: Dict[str, Any] = { - "schema_version": "platform-gates-1.0", - "gates": { - "scenario": { - "enabled": True, - "ready": False, # Phase 5/6 - "thresholds": {"min_scenario_types_covered": 5}, - "message": "TODO(Phase 5): scenario coverage vs ODDDefinition", - }, - "coverage": { - "enabled": True, - "ready": False, - "thresholds": {"min_cohort_support": 200, "min_dim_coverage": 0.8}, - "message": "TODO(Phase 5): ODD combinatorial coverage via MegaEval cube", - }, - "regression": { - "enabled": True, - "ready": True, # can use MegaEval compare / LabelEval regression when run_ids provided - "thresholds": { - "max_recall_drop": 0.010, - "max_precision_drop": 0.015, - "max_safety_recall_drop": 0.005, - }, - "message": "Uses MegaEval promotion policy when candidate/baseline runs supplied", - }, - "safety": { - "enabled": True, - "ready": False, - "thresholds": {}, # filled in Phase 4 from config only - "message": "TODO(Phase 4): SSAM DRAC/DeltaS conflict engine", - }, - "release": { - "enabled": True, - "ready": False, - "thresholds": {"require_all_prior_passed": 1.0}, - "message": "TODO(Phase 6): release orchestration after Safety+Regression", - }, - "quality": { - "enabled": True, - "ready": True, - "thresholds": { - "map_3d": 0.65, - "orientation_error_deg": 5.0, - "id_swap_rate": 0.02, - "track_fragmentation_rate": 0.05, - "position_error_m": 2.0, - }, - "message": "Delegates to sensorflow.quality_gate.QualityGate when sequence_id set", - }, - "launch": { - "enabled": True, - "ready": True, - "thresholds": { - "map_3d": 0.65, - "orientation_error_deg": 5.0, - "id_swap_rate": 0.02, - "track_fragmentation_rate": 0.05, - }, - "message": "Delegates to LaunchGateEvaluator when sequence_id set", - }, - }, -} - - -def load_gate_config(path: Optional[Path] = None) -> Dict[str, Any]: - path = path or DEFAULT_GATE_CONFIG_PATH - if path.exists(): - with open(path) as f: - cfg = json.load(f) - # merge missing keys from defaults without wiping user thresholds - merged = json.loads(json.dumps(DEFAULT_GATE_CONFIG)) - for gname, gcfg in (cfg.get("gates") or {}).items(): - if gname in merged["gates"]: - merged["gates"][gname].update(gcfg) - else: - merged["gates"][gname] = gcfg - return merged - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w") as f: - json.dump(DEFAULT_GATE_CONFIG, f, indent=2) - return json.loads(json.dumps(DEFAULT_GATE_CONFIG)) - - -def save_gate_config(cfg: Dict[str, Any], path: Optional[Path] = None) -> Path: - path = path or DEFAULT_GATE_CONFIG_PATH - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w") as f: - json.dump(cfg, f, indent=2) - return path - - -GATE_ORDER = ["scenario", "coverage", "regression", "safety", "quality", "launch", "release"] - - -def list_gate_defs(cfg: Optional[Dict[str, Any]] = None) -> List[QualityGateDef]: - cfg = cfg or load_gate_config() - out = [] - for name in GATE_ORDER: - g = cfg["gates"].get(name, {}) - out.append(QualityGateDef( - gate_id=f"gate-{name}", - name=name.title(), - gate_type=name, # type: ignore[arg-type] - thresholds=dict(g.get("thresholds") or {}), - config_path=str(DEFAULT_GATE_CONFIG_PATH), - enabled=bool(g.get("enabled", True)), - version="v1", - provenance=Provenance(source_system="platform", notes=g.get("message", "")), - )) - return out - - -def evaluate_multi_gates( - *, - sequence_id: Optional[str] = None, - candidate_run_id: Optional[str] = None, - baseline_run_id: Optional[str] = None, - config_path: Optional[Path] = None, -) -> Dict[str, Any]: - cfg = load_gate_config(config_path) - results: List[GateResult] = [] - - for name in GATE_ORDER: - g = cfg["gates"][name] - if not g.get("enabled", True): - continue - ready = bool(g.get("ready", False)) - thresholds = dict(g.get("thresholds") or {}) - result = GateResult( - result_id=_id("gateres"), - gate_id=f"gate-{name}", - gate_type=name, - thresholds=thresholds, - ready=ready, - message=g.get("message", ""), - scope_ref=sequence_id or candidate_run_id, - passed=None, - ) - - if name == "quality" and sequence_id and ready: - result = _eval_quality(sequence_id, thresholds, result) - elif name == "launch" and sequence_id and ready: - result = _eval_launch(sequence_id, result) - elif name == "regression" and candidate_run_id and baseline_run_id and ready: - result = _eval_regression(candidate_run_id, baseline_run_id, thresholds, result) - elif not ready: - result.passed = None - result.message = g.get("message") or f"{name} gate skeleton — not yet wired" - else: - result.passed = None - result.message = f"{name} gate ready but missing required inputs" - - results.append(result) - - # Overall readiness: all ready gates that produced a boolean must pass; - # unwired gates block "release ready" but do not fail the suite silently. - wired = [r for r in results if r.ready and r.passed is not None] - unwired = [r for r in results if not r.ready] - all_wired_passed = all(r.passed for r in wired) if wired else False - - return { - "config_path": str(config_path or DEFAULT_GATE_CONFIG_PATH), - "sequence_id": sequence_id, - "candidate_run_id": candidate_run_id, - "baseline_run_id": baseline_run_id, - "gates": [r.model_dump() for r in results], - "summary": { - "wired_count": len(wired), - "unwired_count": len(unwired), - "wired_all_passed": all_wired_passed, - "release_ready": False, # Phase 6 - "unwired_gate_types": [r.gate_type for r in unwired], - }, - } - - -def _eval_quality(sequence_id: str, thresholds: Dict[str, float], result: GateResult) -> GateResult: - try: - from sensorflow.quality_gate import QualityGate - from sensorflow.schemas.unified_frame import UnifiedSequence - - manifest = Path("runs/pipeline") / sequence_id / "manifest.json" - tracks_path = Path("runs/pipeline") / sequence_id / "tracks.json" - if not manifest.exists() or not tracks_path.exists(): - result.passed = False - result.failures = [{"metric": "artifacts", "message": "manifest/tracks missing"}] - result.message = "Quality gate: run ingest/auto-label/track first" - return result - sequence = UnifiedSequence.load(manifest) - pred_tracks = json.loads(tracks_path.read_text()) - gate = QualityGate(thresholds=thresholds) - out = gate.evaluate(sequence, pred_tracks) - result.passed = bool(out.get("passed")) - result.metrics = out.get("metric_card") or {} - result.failures = out.get("quality_report", {}).get("failures") or out.get("failures") or [] - result.message = "QualityGate evaluated" - except Exception as exc: - result.passed = False - result.failures = [{"error": str(exc)}] - result.message = f"Quality gate error: {exc}" - return result - - -def _eval_launch(sequence_id: str, result: GateResult) -> GateResult: - try: - from sensorflow.launch_gate_evaluator import LaunchGateEvaluator - evaluator = LaunchGateEvaluator() - out = evaluator.evaluate(sequence_id) - result.passed = bool(out.get("passed")) - result.failures = out.get("failures") or [] - result.metrics = out.get("metrics") or {} - result.thresholds = evaluator.thresholds - result.message = "LaunchGateEvaluator evaluated" - except Exception as exc: - result.passed = False - result.failures = [{"error": str(exc)}] - result.message = f"Launch gate error: {exc}" - return result - - -def _eval_regression( - candidate_run_id: str, - baseline_run_id: str, - thresholds: Dict[str, float], - result: GateResult, -) -> GateResult: - try: - from sensorflow.platform.compare import compare_models - cmp = compare_models( - [baseline_run_id, candidate_run_id], - baseline_run_id=baseline_run_id, - policy=thresholds, - ) - pair = (cmp.get("pairwise") or [{}])[0] - rec = pair.get("recommendation") - result.passed = rec == "PROMOTE" - result.failures = [{"blocker": b} for b in (pair.get("blockers") or [])] - result.metrics = {"recommendation": rec, "headline": pair.get("headline")} - result.message = f"Regression gate: {rec}" - except Exception as exc: - result.passed = False - result.failures = [{"error": str(exc)}] - result.message = f"Regression gate error: {exc}" - return result diff --git a/hf-space/sensorflow/platform/levels.py b/hf-space/sensorflow/platform/levels.py deleted file mode 100644 index 5b2561f19620fb623f67b3653f175ea275dacc7e..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/platform/levels.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Evaluation abstraction across aggregate levels. - -Unifies LabelEval (frame/annotation) and MegaEval (container/cohort/population) -into one ladder without duplicating storage. -""" - -from __future__ import annotations - -from enum import Enum -from typing import Any, Dict, List, Optional - -from pydantic import BaseModel, Field - - -class AggregateLevel(str, Enum): - FRAME = "frame" - CLIP = "clip" # sequence / short clip - SCENE = "scene" - DRIVE = "drive" - CONTAINER = "container" - DATASET = "dataset" - COHORT = "cohort" - POPULATION = "population" - - @classmethod - def ladder(cls) -> List["AggregateLevel"]: - return [ - cls.FRAME, - cls.CLIP, - cls.SCENE, - cls.DRIVE, - cls.CONTAINER, - cls.DATASET, - cls.COHORT, - cls.POPULATION, - ] - - def parent(self) -> Optional["AggregateLevel"]: - ladder = self.ladder() - i = ladder.index(self) - return ladder[i + 1] if i + 1 < len(ladder) else None - - def child(self) -> Optional["AggregateLevel"]: - ladder = self.ladder() - i = ladder.index(self) - return ladder[i - 1] if i > 0 else None - - -# Which backend owns primary aggregates at each level -LEVEL_BACKEND: Dict[AggregateLevel, str] = { - AggregateLevel.FRAME: "labeleval", - AggregateLevel.CLIP: "labeleval", - AggregateLevel.SCENE: "labeleval", - AggregateLevel.DRIVE: "platform", # stub until Phase 5/6 - AggregateLevel.CONTAINER: "megaeval", - AggregateLevel.DATASET: "labeleval", - AggregateLevel.COHORT: "megaeval", - AggregateLevel.POPULATION: "megaeval", -} - - -class EvaluationScope(BaseModel): - """Points at one aggregate cell without inventing a second DB.""" - - level: AggregateLevel - evaluation_id: Optional[str] = None - dataset_id: Optional[str] = None - population_id: Optional[str] = None - run_id: Optional[str] = None - container_id: Optional[str] = None - mega_container_id: Optional[int] = None - drive_id: Optional[str] = None - scene_id: Optional[str] = None - sequence_id: Optional[str] = None - frame_id: Optional[str] = None - cohort_filters: Dict[str, List[str]] = Field(default_factory=dict) - model_version: Optional[str] = None - label_version: Optional[str] = None - - def backend(self) -> str: - return LEVEL_BACKEND[self.level] - - def drill_down_hint(self) -> AggregateLevel: - child = self.level.child() - return child or self.level - - -def summarize_scope(scope: EvaluationScope) -> Dict[str, Any]: - return { - "level": scope.level.value, - "backend": scope.backend(), - "parent_level": scope.level.parent().value if scope.level.parent() else None, - "child_level": scope.level.child().value if scope.level.child() else None, - "refs": { - k: v - for k, v in { - "evaluation_id": scope.evaluation_id, - "dataset_id": scope.dataset_id, - "population_id": scope.population_id, - "run_id": scope.run_id, - "container_id": scope.container_id, - "mega_container_id": scope.mega_container_id, - "drive_id": scope.drive_id, - "scene_id": scope.scene_id, - "sequence_id": scope.sequence_id, - "frame_id": scope.frame_id, - "model_version": scope.model_version, - }.items() - if v is not None - }, - "cohort_filters": scope.cohort_filters, - } diff --git a/hf-space/sensorflow/platform/metrics_engine.py b/hf-space/sensorflow/platform/metrics_engine.py deleted file mode 100644 index 492b20e561848698ebf1483f9160c42340102a16..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/platform/metrics_engine.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Metric engine: precision/recall/F1/IoU + verification rates. - -Pure functions — no I/O. Callers feed counts from MegaEval containers or -LabelEval match stats. -""" - -from __future__ import annotations - -from typing import Any, Dict, Iterable, List, Optional, Sequence - - -def safe_div(num: float, den: float) -> Optional[float]: - if den <= 0: - return None - return num / den - - -def round4(v: Optional[float]) -> Optional[float]: - return None if v is None else round(float(v), 4) - - -def precision_recall_f1(tp: int, fp: int, fn: int) -> Dict[str, Optional[float]]: - precision = safe_div(tp, tp + fp) - recall = safe_div(tp, tp + fn) - f1 = None - if precision is not None and recall is not None and (precision + recall) > 0: - f1 = 2 * precision * recall / (precision + recall) - return { - "precision": round4(precision), - "recall": round4(recall), - "f1": round4(f1), - "tp": tp, - "fp": fp, - "fn": fn, - } - - -def mean_iou(sum_iou: float, tp: int) -> Optional[float]: - return round4(safe_div(sum_iou, tp)) - - -def verification_rates( - *, - n_objects: int, - verified: int = 0, - reviewed: int = 0, - disputed: int = 0, - auto_accepted: int = 0, - hitl: int = 0, - unverified: Optional[int] = None, -) -> Dict[str, Any]: - """Verification / HITL funnel rates at container (or dataset) scope. - - unverified defaults to n_objects - verified when not provided. - """ - n = max(0, int(n_objects)) - verified = max(0, int(verified)) - reviewed = max(0, int(reviewed)) - disputed = max(0, int(disputed)) - auto_accepted = max(0, int(auto_accepted)) - hitl = max(0, int(hitl)) - if unverified is None: - unverified = max(0, n - verified) - - def rate(count: int) -> Optional[float]: - return round4(safe_div(count, n)) if n > 0 else None - - return { - "n_objects": n, - "verified": verified, - "unverified": unverified, - "disputed": disputed, - "auto_accepted": auto_accepted, - "hitl": hitl, - "reviewed": reviewed, - "verified_rate": rate(verified), - "unverified_rate": rate(unverified), - "disputed_rate": rate(disputed), - "auto_accept_rate": rate(auto_accepted), - "hitl_rate": rate(hitl), - "review_coverage": rate(reviewed), - } - - -def container_quality_metrics( - *, - tp: int, - fp: int, - fn: int, - sum_iou: float = 0.0, - n_objects: int = 0, - verified: int = 0, - reviewed: int = 0, - disputed: int = 0, - auto_accepted: int = 0, - hitl: int = 0, - anomalies: int = 0, -) -> Dict[str, Any]: - prf = precision_recall_f1(tp, fp, fn) - n = n_objects or (tp + fp) - ver = verification_rates( - n_objects=n, - verified=verified, - reviewed=reviewed, - disputed=disputed, - auto_accepted=auto_accepted, - hitl=hitl, - ) - return { - **prf, - "mean_iou": mean_iou(sum_iou, tp), - "anomalies": anomalies, - "anomaly_rate": round4(safe_div(anomalies, n)) if n else None, - "verification": ver, - } - - -def aggregate_container_rows(rows: Sequence[Dict[str, Any]]) -> Dict[str, Any]: - """Roll up a page of container metric dicts into one profile summary.""" - tp = fp = fn = verified = reviewed = disputed = auto_acc = hitl = anomalies = n = 0 - sum_iou = 0.0 - for r in rows: - tp += int(r.get("tp") or 0) - fp += int(r.get("fp") or 0) - fn += int(r.get("fn") or 0) - verified += int(r.get("verified") or 0) - reviewed += int(r.get("reviewed") or 0) - disputed += int(r.get("disputed") or 0) - auto_acc += int(r.get("auto_accepted") or 0) - hitl += int(r.get("hitl") or 0) - anomalies += int(r.get("anomalies") or 0) - n += int(r.get("n_objects") or 0) - mi = r.get("mean_iou") - tpi = int(r.get("tp") or 0) - if mi is not None and tpi > 0: - sum_iou += float(mi) * tpi - elif r.get("sum_iou") is not None: - sum_iou += float(r["sum_iou"]) - return container_quality_metrics( - tp=tp, fp=fp, fn=fn, sum_iou=sum_iou, n_objects=n, - verified=verified, reviewed=reviewed, disputed=disputed, - auto_accepted=auto_acc, hitl=hitl, anomalies=anomalies, - ) - - -def delta_metrics( - baseline: Dict[str, Optional[float]], - candidate: Dict[str, Optional[float]], - keys: Optional[Iterable[str]] = None, -) -> List[Dict[str, Any]]: - keys = list(keys or ("precision", "recall", "f1", "mean_iou", "safety_recall", "anomaly_rate")) - out = [] - for k in keys: - b, c = baseline.get(k), candidate.get(k) - if b is None or c is None: - continue - out.append({ - "metric": k, - "baseline": round4(float(b)), - "candidate": round4(float(c)), - "delta": round4(float(c) - float(b)), - }) - return out diff --git a/hf-space/sensorflow/quality_gate.py b/hf-space/sensorflow/quality_gate.py deleted file mode 100644 index dd4d7425e23e3b7455dd5880bd08a82a4203dd4b..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/quality_gate.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Quality gate benchmarking against vendor ground truth.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - -from sensorflow.metrics.perception_3d import ( - bev_iou, - compute_map_mar, - mean_orientation_error, - mean_position_error, -) -from sensorflow.metrics.resource_profile import compute_resource_profile -from sensorflow.metrics.temporal_mot import ( - compute_id_swap_rate, - compute_track_fragmentation_rate, -) -from sensorflow.schemas.unified_frame import UnifiedSequence - - -DEFAULT_THRESHOLDS = { - "map_3d": 0.65, - "orientation_error_deg": 5.0, - "id_swap_rate": 0.02, - "track_fragmentation_rate": 0.05, - "position_error_m": 2.0, -} - - -class QualityGate: - """Benchmark automated tracks against vendor ground truth.""" - - def __init__(self, thresholds: Optional[Dict[str, float]] = None): - self.thresholds = thresholds or DEFAULT_THRESHOLDS.copy() - - def evaluate( - self, - sequence: UnifiedSequence, - pred_tracks: List[Dict], - stages_run: Optional[List[str]] = None, - ) -> Dict[str, Any]: - all_preds, all_gts = self._collect_boxes(sequence, pred_tracks) - map_metrics = compute_map_mar(all_preds, all_gts) - - matched_pairs = self._match_for_errors(all_preds, all_gts) - orient_err = mean_orientation_error(matched_pairs) - pos_err = mean_position_error(matched_pairs) - - gt_tracks = self._build_gt_tracks(sequence) - id_swap = compute_id_swap_rate(pred_tracks, gt_tracks) - fragmentation = compute_track_fragmentation_rate(pred_tracks, gt_tracks) - - resource = compute_resource_profile(sequence.frames, stages_run) - - metric_card = { - "map_3d": map_metrics["map_3d"], - "mar_3d": map_metrics["mar_3d"], - "mean_iou_3d": map_metrics["mean_iou_3d"], - "orientation_error_deg": orient_err, - "position_error_m": pos_err, - "id_swap_rate": id_swap, - "track_fragmentation_rate": fragmentation, - "process_units": resource["process_units"], - "compute_cycles": resource["compute_cycles"], - } - - failures = self._check_thresholds(metric_card) - passed = len(failures) == 0 - - quality_report = { - "total_predictions": len(all_preds), - "total_ground_truth": len(all_gts), - "quality_score": self._quality_score(metric_card), - "metric_card": metric_card, - "passed": passed, - "failures": failures, - "issue_summary": { - "high_orientation_error": 1 if orient_err > self.thresholds["orientation_error_deg"] else 0, - "high_position_error": 1 if pos_err > self.thresholds["position_error_m"] else 0, - "id_swaps": 1 if id_swap > self.thresholds["id_swap_rate"] else 0, - "fragmentation": 1 if fragmentation > self.thresholds["track_fragmentation_rate"] else 0, - }, - "recommendations": self._recommendations(metric_card, failures), - } - - return {"metric_card": metric_card, "quality_report": quality_report, "passed": passed} - - def _collect_boxes( - self, - sequence: UnifiedSequence, - pred_tracks: List[Dict], - ) -> Tuple[List[List[float]], List[List[float]]]: - all_preds, all_gts = [], [] - for track in pred_tracks: - for fr in track.get("frames", []): - all_preds.append(fr["bbox_3d"]) - for frame in sequence.frames: - for gt in frame.ground_truth: - all_gts.append(gt.bbox_3d) - return all_preds, all_gts - - def _match_for_errors( - self, - preds: List[List[float]], - gts: List[List[float]], - ) -> List[Tuple[List[float], List[float]]]: - pairs = [] - used_gt = set() - for pred in preds: - best_iou, best_gi = 0, -1 - for gi, gt in enumerate(gts): - if gi in used_gt: - continue - iou = bev_iou(pred, gt) - if iou > best_iou: - best_iou = iou - best_gi = gi - if best_gi >= 0 and best_iou > 0.3: - pairs.append((pred, gts[best_gi])) - used_gt.add(best_gi) - return pairs - - def _build_gt_tracks(self, sequence: UnifiedSequence) -> List[Dict]: - by_instance: Dict[str, List[Dict]] = {} - for frame in sequence.frames: - for gt in frame.ground_truth: - by_instance.setdefault(gt.instance_id, []).append({ - "frame_id": frame.frame_id, - "bbox_3d": gt.bbox_3d, - }) - return [ - {"instance_id": iid, "frames": frames} - for iid, frames in by_instance.items() - ] - - def _check_thresholds(self, metrics: Dict[str, float]) -> List[Dict]: - failures = [] - checks = [ - ("map_3d", metrics["map_3d"], self.thresholds["map_3d"], ">="), - ("orientation_error_deg", metrics["orientation_error_deg"], self.thresholds["orientation_error_deg"], "<="), - ("id_swap_rate", metrics["id_swap_rate"], self.thresholds["id_swap_rate"], "<="), - ("track_fragmentation_rate", metrics["track_fragmentation_rate"], self.thresholds["track_fragmentation_rate"], "<="), - ("position_error_m", metrics["position_error_m"], self.thresholds["position_error_m"], "<="), - ] - for name, value, threshold, op in checks: - failed = (value < threshold) if op == ">=" else (value > threshold) - if failed: - failures.append({ - "metric": name, - "value": value, - "threshold": threshold, - "operator": op, - }) - return failures - - def _quality_score(self, metrics: Dict[str, float]) -> float: - def safe(v, default=0.0): - import math - return default if v is None or (isinstance(v, float) and math.isnan(v)) else v - - score = ( - safe(metrics.get("map_3d", 0)) * 40 + - (1 - min(safe(metrics.get("orientation_error_deg", 0)) / 45, 1)) * 20 + - (1 - min(safe(metrics.get("id_swap_rate", 0)) / 0.1, 1)) * 20 + - (1 - min(safe(metrics.get("track_fragmentation_rate", 0)) / 0.2, 1)) * 20 - ) - return round(min(100, max(0, score)), 1) - - def _recommendations(self, metrics: Dict, failures: List[Dict]) -> List[str]: - recs = [] - if metrics["orientation_error_deg"] > self.thresholds["orientation_error_deg"]: - recs.append("Improve 3D orientation estimation via tighter SAM mask-to-LiDAR alignment.") - if metrics["id_swap_rate"] > self.thresholds["id_swap_rate"]: - recs.append("Increase tracker velocity penalty or reduce association gate during occlusions.") - if metrics["track_fragmentation_rate"] > self.thresholds["track_fragmentation_rate"]: - recs.append("Extend max_age for ghost tracks and review occlusion handling.") - if metrics["map_3d"] < self.thresholds["map_3d"]: - recs.append("Retrain or tune perception model; verify LiDAR-camera calibration.") - if not recs: - recs.append("All quality gate thresholds met. Proceed to launch gate.") - return recs - - def save_results(self, sequence_id: str, results: Dict[str, Any]) -> Path: - out_dir = Path("runs/pipeline") / sequence_id / "benchmark" - out_dir.mkdir(parents=True, exist_ok=True) - with open(out_dir / "metric_card.json", "w") as f: - json.dump(results["metric_card"], f, indent=2) - with open(out_dir / "quality_report.json", "w") as f: - json.dump(results["quality_report"], f, indent=2) - return out_dir diff --git a/hf-space/sensorflow/schemas/__init__.py b/hf-space/sensorflow/schemas/__init__.py deleted file mode 100644 index a9dee12fef768be722148f81548a112db78a79ce..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/schemas/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from sensorflow.schemas.unified_frame import * -from sensorflow.schemas.taxonomy_axes import TaxonomyAxes, assign_taxonomy_axes diff --git a/hf-space/sensorflow/schemas/taxonomy_axes.py b/hf-space/sensorflow/schemas/taxonomy_axes.py deleted file mode 100644 index 9df3f5202949cf91488d9549a1206e1cef05624d..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/schemas/taxonomy_axes.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Six-axis failure taxonomy for stratification.""" - -from __future__ import annotations - -from typing import Optional - -from pydantic import BaseModel, Field - - -class TaxonomyAxes(BaseModel): - """Six-axis failure taxonomy: Mode, Polarity, Actor, Vulnerable, Legality, Infra.""" - - mode: str = "urban" - polarity: str = "benign" - actor: str = "unknown" - vulnerable: bool = False - legality: str = "legal" - infra: str = "standard" - - -VULNERABLE_ACTORS = { - "pedestrian", "cyclist", "motorcyclist", "child", "wheelchair", - "person", "skater", "scooter", -} - -INFRA_CLASSES = { - "construction", "barrier", "traffic_cone", "traffic_light", - "stop_sign", "temporary", -} - -ILLEGAL_CONTEXTS = {"jaywalking", "wrong_way", "red_light_run"} - - -def assign_taxonomy_axes( - class_name: str, - speed_kmh: float = 0.0, - context: Optional[str] = None, -) -> TaxonomyAxes: - """Derive six-axis taxonomy tags from class name and scene context.""" - actor = class_name.lower().replace(" ", "_") - vulnerable = actor in VULNERABLE_ACTORS - infra = "temporary" if actor in INFRA_CLASSES else "standard" - - if speed_kmh > 80: - mode = "highway" - elif speed_kmh > 40: - mode = "arterial" - else: - mode = "urban" - - polarity = "adversarial" if context in {"occluded", "edge_case", "rare"} else "benign" - legality = "illegal" if context in ILLEGAL_CONTEXTS else "legal" - - return TaxonomyAxes( - mode=mode, - polarity=polarity, - actor=actor, - vulnerable=vulnerable, - legality=legality, - infra=infra, - ) diff --git a/hf-space/sensorflow/schemas/unified_frame.py b/hf-space/sensorflow/schemas/unified_frame.py deleted file mode 100644 index 8e239275290efbbe32119b2f01344aa2ba157e54..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/schemas/unified_frame.py +++ /dev/null @@ -1,78 +0,0 @@ -"""OpenLABEL-inspired unified frame/sequence schema.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, Dict, List, Literal, Optional - -from pydantic import BaseModel, Field - -from sensorflow.schemas.taxonomy_axes import TaxonomyAxes - - -class EgoPose(BaseModel): - x: float = 0.0 - y: float = 0.0 - z: float = 0.0 - yaw: float = 0.0 - speed_kmh: float = 0.0 - - -class LidarData(BaseModel): - path: Optional[str] = None - num_points: int = 0 - format: str = "bin" - - -class CameraView(BaseModel): - image_path: str - intrinsics: List[List[float]] = Field(default_factory=lambda: [[1, 0, 0], [0, 1, 0], [0, 0, 1]]) - extrinsics: List[List[float]] = Field(default_factory=lambda: [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) - - -class Calibration(BaseModel): - lidar_to_ego: List[List[float]] = Field(default_factory=lambda: [[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) - camera_to_ego: Dict[str, List[List[float]]] = Field(default_factory=dict) - - -class Object3D(BaseModel): - bbox_3d: List[float] = Field(description="[x, y, z, l, w, h, yaw]") - class_name: str = "unknown" - confidence: float = 0.0 - track_id: Optional[int] = None - sam_mask_ref: Optional[str] = None - taxonomy_axes: TaxonomyAxes = Field(default_factory=TaxonomyAxes) - object_id: Optional[str] = None - - -class GroundTruthObject(Object3D): - instance_id: str = "" - - -class FusedFrame(BaseModel): - frame_id: str - timestamp_us: int = 0 - lidar: LidarData = Field(default_factory=LidarData) - cameras: Dict[str, CameraView] = Field(default_factory=dict) - ego_pose: EgoPose = Field(default_factory=EgoPose) - proposals: List[Object3D] = Field(default_factory=list) - ground_truth: List[GroundTruthObject] = Field(default_factory=list) - - -class UnifiedSequence(BaseModel): - sequence_id: str - vendor: Literal["alpamayo", "waymo", "mixed"] = "mixed" - frames: List[FusedFrame] = Field(default_factory=list) - calibration: Calibration = Field(default_factory=Calibration) - taxonomy_manifest: Dict[str, Any] = Field(default_factory=dict) - - def save(self, path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w") as f: - json.dump(self.model_dump(), f, indent=2) - - @classmethod - def load(cls, path: Path) -> "UnifiedSequence": - with open(path) as f: - return cls.model_validate(json.load(f)) diff --git a/hf-space/sensorflow/temporal_tracker.py b/hf-space/sensorflow/temporal_tracker.py deleted file mode 100644 index 1bc445cd08039b2ffa4ccc68c0ad2f77db4fc41c..0000000000000000000000000000000000000000 --- a/hf-space/sensorflow/temporal_tracker.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Temporal tracking with Kalman filter and Hungarian association.""" - -from __future__ import annotations - -import json -import math -from dataclasses import dataclass, field -from pathlib import Path -from typing import Dict, List, Optional, Tuple - -import numpy as np - -from sensorflow.schemas.unified_frame import Object3D - - -@dataclass -class KalmanTrack: - track_id: int - state: np.ndarray # [x, y, vx, vy] - covariance: np.ndarray - missed_frames: int = 0 - history: List[Dict] = field(default_factory=list) - class_name: str = "unknown" - confidence: float = 0.0 - - def predict(self, dt: float = 0.1): - F = np.array([[1, 0, dt, 0], [0, 1, 0, dt], [0, 0, 1, 0], [0, 0, 0, 1]]) - Q = np.eye(4) * 0.1 - self.state = F @ self.state - self.covariance = F @ self.covariance @ F.T + Q - - def update(self, measurement: np.ndarray): - H = np.array([[1, 0, 0, 0], [0, 1, 0, 0]]) - R = np.eye(2) * 0.5 - y = measurement - H @ self.state - S = H @ self.covariance @ H.T + R - K = self.covariance @ H.T @ np.linalg.inv(S) - self.state = self.state + K @ y - self.covariance = (np.eye(4) - K @ H) @ self.covariance - self.missed_frames = 0 - - -class TemporalTracker: - """Kalman + Hungarian tracker for 3D bounding box sequences.""" - - def __init__( - self, - max_age: int = 30, - distance_gate: float = 3.0, - velocity_penalty: float = 2.0, - ): - self.max_age = max_age - self.distance_gate = distance_gate - self.velocity_penalty = velocity_penalty - self.tracks: Dict[int, KalmanTrack] = {} - self.next_id = 1 - self.id_swap_log: List[Dict] = [] - - def update(self, frame_id: str, proposals: List[Object3D]) -> List[Object3D]: - for track in self.tracks.values(): - track.predict() - track.missed_frames += 1 - - if proposals and self.tracks: - assignments = self._associate(proposals) - matched_proposals = set() - for track_id, prop_idx in assignments: - prop = proposals[prop_idx] - track = self.tracks[track_id] - measurement = np.array([prop.bbox_3d[0], prop.bbox_3d[1]]) - track.update(measurement) - track.class_name = prop.class_name - track.confidence = prop.confidence - track.history.append({ - "frame_id": frame_id, - "bbox_3d": prop.bbox_3d, - "confidence": prop.confidence, - }) - matched_proposals.add(prop_idx) - - for i, prop in enumerate(proposals): - if i not in matched_proposals: - self._spawn_track(frame_id, prop) - elif proposals: - for prop in proposals: - self._spawn_track(frame_id, prop) - - dead = [tid for tid, t in self.tracks.items() if t.missed_frames > self.max_age] - for tid in dead: - del self.tracks[tid] - - result = [] - for track in self.tracks.values(): - if track.history and track.history[-1]["frame_id"] == frame_id: - prop = Object3D( - bbox_3d=track.history[-1]["bbox_3d"], - class_name=track.class_name, - confidence=track.confidence, - track_id=track.track_id, - ) - result.append(prop) - return result - - def _associate(self, proposals: List[Object3D]) -> List[Tuple[int, int]]: - try: - from scipy.optimize import linear_sum_assignment - except ImportError: - return self._greedy_associate(proposals) - - track_ids = list(self.tracks.keys()) - cost = np.full((len(track_ids), len(proposals)), 1e6) - - for ti, tid in enumerate(track_ids): - track = self.tracks[tid] - for pi, prop in enumerate(proposals): - dist = math.sqrt( - (track.state[0] - prop.bbox_3d[0]) ** 2 + - (track.state[1] - prop.bbox_3d[1]) ** 2 - ) - yaw_diff = abs(track.state[2] if len(track.state) > 2 else 0 - prop.bbox_3d[6]) - vel_penalty = self.velocity_penalty * yaw_diff - cost[ti, pi] = dist + vel_penalty - - row_ind, col_ind = linear_sum_assignment(cost) - assignments = [] - for r, c in zip(row_ind, col_ind): - if cost[r, c] < self.distance_gate: - assignments.append((track_ids[r], c)) - return assignments - - def _greedy_associate(self, proposals: List[Object3D]) -> List[Tuple[int, int]]: - assignments = [] - used = set() - for tid, track in self.tracks.items(): - best_pi, best_dist = None, float("inf") - for pi, prop in enumerate(proposals): - if pi in used: - continue - dist = math.sqrt( - (track.state[0] - prop.bbox_3d[0]) ** 2 + - (track.state[1] - prop.bbox_3d[1]) ** 2 - ) - if dist < best_dist and dist < self.distance_gate: - best_dist = dist - best_pi = pi - if best_pi is not None: - assignments.append((tid, best_pi)) - used.add(best_pi) - return assignments - - def _spawn_track(self, frame_id: str, prop: Object3D): - state = np.array([prop.bbox_3d[0], prop.bbox_3d[1], 0.0, 0.0]) - track = KalmanTrack( - track_id=self.next_id, - state=state, - covariance=np.eye(4), - class_name=prop.class_name, - confidence=prop.confidence, - ) - track.history.append({ - "frame_id": frame_id, - "bbox_3d": prop.bbox_3d, - "confidence": prop.confidence, - }) - self.tracks[self.next_id] = track - self.next_id += 1 - - def run_sequence( - self, - proposals_by_frame: Dict[str, List[Object3D]], - output_path: Path, - ) -> List[Dict]: - for frame_id in sorted(proposals_by_frame.keys()): - self.update(frame_id, proposals_by_frame[frame_id]) - - tracks_output = [] - for tid, track in self.tracks.items(): - if track.history: - tracks_output.append({ - "track_id": tid, - "class_name": track.class_name, - "frames": track.history, - }) - - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "w") as f: - json.dump(tracks_output, f, indent=2) - return tracks_output - - @staticmethod - def load_proposals(proposals_dir: Path) -> Dict[str, List[Object3D]]: - result = {} - for p in sorted(proposals_dir.glob("*.json")): - frame_id = p.stem - with open(p) as f: - data = json.load(f) - result[frame_id] = [Object3D.model_validate(d) for d in data] - return result diff --git a/hf-space/skills/README.md b/hf-space/skills/README.md deleted file mode 100644 index 864dd3143a65b90d955f7af4213961b37ce3321e..0000000000000000000000000000000000000000 --- a/hf-space/skills/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# Reusable Analytics Agent Skills - -These 12 skills are distilled from the supplied codebase while intentionally excluding domain-specific references and internal terminology. - -## Skills - -1. grounded-data-agent -2. natural-language-ui-controller -3. schema-aware-analytics -4. investigative-analytics-workspace -5. large-scale-analytics-engine -6. llm-provider-orchestration -7. streaming-llm-interface -8. llm-evaluation-feedback -9. automated-report-generation -10. ml-observability -11. human-vs-ai-evaluation -12. enterprise-analytics-ux - -Each skill is self-contained in `/SKILL.md`. - -The skills are designed to compose: -- grounded-data-agent provides evidence discipline -- natural-language-ui-controller provides agentic UI actions -- schema-aware-analytics provides data-model adaptation -- investigative-analytics-workspace provides the end-user investigation loop -- large-scale-analytics-engine provides scalable retrieval -- llm-provider-orchestration and streaming-llm-interface provide inference infrastructure -- evaluation, reporting, observability, and UX skills provide production operating capabilities diff --git a/hf-space/skills/automated-report-generation/SKILL.md b/hf-space/skills/automated-report-generation/SKILL.md deleted file mode 100644 index ddd0d2232b09dc7a310aa012cd265ef48ccaddf3..0000000000000000000000000000000000000000 --- a/hf-space/skills/automated-report-generation/SKILL.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: automated-report-generation -description: Generate structured analytical reports asynchronously from approved data and reusable report sections. ---- - -# automated-report-generation - -## Purpose - -Turn recurring analytical workflows into repeatable reports rather than manual copy/paste. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Define report sections and their required data contracts. -- Select the requested sections and analytical scope. -- Retrieve approved data and aggregates. -- Generate each section from grounded evidence. -- Run validation checks on generated content. -- Persist report/job status for asynchronous retrieval. -- Allow sections to be reordered, copied, or exported. - -## Guardrails - -- Reports must be grounded in approved data. -- Separate data retrieval from narrative generation. -- Expose job state and failures. -- Do not claim conclusions that are absent from the supplied evidence. - -## Expected outputs - -- report job -- sectioned analytical report -- job status -- Markdown/export representation - -## Example - -A user selects executive summary, trends, quality, and root-cause sections. The system runs the report asynchronously and makes completed sections available progressively. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/skills/enterprise-analytics-ux/SKILL.md b/hf-space/skills/enterprise-analytics-ux/SKILL.md deleted file mode 100644 index 0cae04149d70e4b326c56331f1191b678ae21ab6..0000000000000000000000000000000000000000 --- a/hf-space/skills/enterprise-analytics-ux/SKILL.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -name: enterprise-analytics-ux -description: Design enterprise analytical interfaces that make complex data, AI interactions, loading states, filters, and investigation context usable and reproducible. ---- - -# enterprise-analytics-ux - -## Purpose - -Provide an operational UX that remains understandable under large datasets, long-running queries, and AI uncertainty. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Establish a consistent application shell and navigation model. -- Use persistent, visible filters and shareable state. -- Provide skeletons and explicit loading/error states. -- Use progressive disclosure for dense analytical information. -- Make evidence and data provenance visible. -- Support keyboard and assistive-technology interaction. -- Keep dark/light themes and design tokens consistent. -- Deep-link analytical views so investigations are reproducible. - -## Guardrails - -- Do not hide important system state behind animation. -- Every major analytical surface needs clear loading, empty, error, and stale-data states. -- Use accessible semantics for dialogs, controls, tables, and navigation. -- Keep high-density dashboards visually hierarchical. -- Do not sacrifice evidence visibility for visual polish. - -## Expected outputs - -- enterprise analytics UX system -- design patterns -- accessibility checklist -- state/deep-link model -- loading/error conventions - -## Example - -A dense analytical dashboard combines persistent filters, KPI summaries, drill-down tables, progressive loading, keyboard-accessible detail views, and URL-addressable investigation state. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/skills/grounded-data-agent/SKILL.md b/hf-space/skills/grounded-data-agent/SKILL.md deleted file mode 100644 index 184d277b6fd227e9349d8ddf19114a53c95bb027..0000000000000000000000000000000000000000 --- a/hf-space/skills/grounded-data-agent/SKILL.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -name: grounded-data-agent -description: Build agents that answer analytical questions using only retrieved, structured evidence and explicit data-grounding contracts. ---- - -# grounded-data-agent - -## Purpose - -Turn natural-language analytical questions into evidence-backed answers without allowing the model to invent facts or directly execute arbitrary SQL. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Parse the user's question into intent, entities, metrics, time range, and constraints. -- Resolve the request into a typed filter/action schema. -- Retrieve data through application-owned, parameterized queries or approved data services. -- Return compact aggregates and relevant evidence to the model. -- Generate the answer only from the supplied evidence. -- Quantify claims when the data supports them and explicitly state insufficiency when it does not. -- Optionally return a structured deep-link or follow-up action. - -## Guardrails - -- Never fabricate a metric, row, dimension value, or trend. -- Do not let the LLM generate executable SQL when an application-owned query layer can do the job. -- Keep factual grounding separate from presentation language. -- Validate all model-generated structured actions before execution. - -## Expected outputs - -- grounded answer -- evidence/aggregate payload -- validated action or deep link when applicable - -## Example - -User: 'Is category A getting worse over the last 12 weeks?' The agent resolves the time window and category, retrieves approved aggregates, compares the relevant metrics, and answers only from those values. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/skills/human-vs-ai-evaluation/SKILL.md b/hf-space/skills/human-vs-ai-evaluation/SKILL.md deleted file mode 100644 index 82a02febc752130b760567c0b285d07bd081fc02..0000000000000000000000000000000000000000 --- a/hf-space/skills/human-vs-ai-evaluation/SKILL.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -name: human-vs-ai-evaluation -description: Measure automated decision quality against human decisions and quantify the transition from manual workflows to AI-assisted or automated workflows. ---- - -# human-vs-ai-evaluation - -## Purpose - -Determine whether automation is actually replacing work safely and effectively. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Define the human reference process and its limitations. -- Define equivalent automated outputs. -- Measure agreement, disagreement, quality, throughput, and latency. -- Segment results by relevant cohorts. -- Track performance over time. -- Investigate disagreement cases rather than hiding them in a single score. -- Use results to decide where automation should expand, pause, or remain human-reviewed. - -## Guardrails - -- Human labels are a reference, not automatically ground truth. -- Report disagreement rates with denominators. -- Separate quality improvement from throughput improvement. -- Track changes to the human process that could invalidate comparisons. - -## Expected outputs - -- human-vs-AI comparison -- agreement metrics -- automation effectiveness -- disagreement analysis -- migration decision signals - -## Example - -An automated classifier processes a workload previously reviewed manually. Compare its outputs with a controlled human sample, quantify agreement and errors, and track whether throughput gains preserve acceptable quality. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/skills/investigative-analytics-workspace/SKILL.md b/hf-space/skills/investigative-analytics-workspace/SKILL.md deleted file mode 100644 index 0757d08470c65f4bed6383776c2d50f380a267cb..0000000000000000000000000000000000000000 --- a/hf-space/skills/investigative-analytics-workspace/SKILL.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -name: investigative-analytics-workspace -description: Design an AI-assisted workspace that connects conversational analysis, filters, aggregate views, and record-level evidence. ---- - -# investigative-analytics-workspace - -## Purpose - -Support an investigation from question to pattern to evidence to follow-up without forcing users to switch tools. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Present an AI command surface alongside structured analytical data. -- Let users apply filters through either UI controls or validated natural-language actions. -- Show aggregate trends and breakdowns. -- Allow drill-down from aggregate results to subsets and individual records. -- Keep the investigation context synchronized across views. -- Support follow-up questions using the current analytical context. - -## Guardrails - -- Every analytical visualization should have an identifiable data source. -- Do not hide the transition from aggregate evidence to individual records. -- Preserve investigation state when moving between views. -- Make loading, errors, and data freshness visible. - -## Expected outputs - -- investigation workspace -- linked aggregate/detail views -- persistent analytical context - -## Example - -A user notices an abnormal trend, selects the affected segment, inspects representative records, and asks the AI to explain the pattern using the same filtered context. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/skills/large-scale-analytics-engine/SKILL.md b/hf-space/skills/large-scale-analytics-engine/SKILL.md deleted file mode 100644 index f795e607484712273251f497cfc04edcd27ff88c..0000000000000000000000000000000000000000 --- a/hf-space/skills/large-scale-analytics-engine/SKILL.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -name: large-scale-analytics-engine -description: Build interactive analytical systems over large warehouse datasets using bounded queries, pre-aggregation, caching, and controlled refresh. ---- - -# large-scale-analytics-engine - -## Purpose - -Keep analytical UX responsive without repeatedly scanning very large fact tables. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Identify fact tables that are too large for per-request interactive aggregation. -- Define the metrics and dimensions required by the UI. -- Create derived or materialized aggregates for those access patterns. -- Use parameterized, bounded queries for interactive requests. -- Cache expensive but reusable results with explicit TTLs. -- Provide explicit refresh behavior for users who need newer data. -- Monitor query latency and revise aggregation strategy when access patterns change. - -## Guardrails - -- Do not make large raw tables the default backend for every chart. -- Prefer small purpose-built aggregates for repeated interactive queries. -- Do not hide stale-cache behavior. -- Keep expensive warehouse work out of the request path when practical. - -## Expected outputs - -- interactive query layer -- pre-aggregated datasets -- cache policy -- refresh strategy -- latency safeguards - -## Example - -A multi-hundred-million-row fact table feeds a weekly trend chart. Build a compact time/dimension aggregate and serve the chart from that aggregate instead of scanning the fact table per request. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/skills/llm-evaluation-feedback/SKILL.md b/hf-space/skills/llm-evaluation-feedback/SKILL.md deleted file mode 100644 index 422620c20a93340566f252f9080bb885fb403475..0000000000000000000000000000000000000000 --- a/hf-space/skills/llm-evaluation-feedback/SKILL.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: llm-evaluation-feedback -description: Capture, aggregate, and use human feedback on LLM outputs to measure production quality. ---- - -# llm-evaluation-feedback - -## Purpose - -Turn subjective user feedback into an observable evaluation loop. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Attach feedback to a specific model response and context. -- Capture positive, negative, or structured review signals. -- Aggregate feedback by output type, model, prompt family, and time period where appropriate. -- Monitor quality trends and recurring failure patterns. -- Use findings to improve prompts, retrieval, schemas, or provider selection. -- Retain representative examples for regression testing. - -## Guardrails - -- Feedback must be traceable to the response being evaluated. -- Do not equate raw thumbs-up rate with complete model quality. -- Separate user preference from factual correctness when possible. -- Protect sensitive response/context data. - -## Expected outputs - -- feedback events -- quality dashboards -- failure taxonomy -- evaluation dataset -- regression cases - -## Example - -Negative feedback clusters around one response type. The team samples those cases, identifies a grounding failure, fixes the retrieval contract, and adds the cases to regression evaluation. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/skills/llm-provider-orchestration/SKILL.md b/hf-space/skills/llm-provider-orchestration/SKILL.md deleted file mode 100644 index 8551343ed2921adf0504f42c2cb760601ba6a77c..0000000000000000000000000000000000000000 --- a/hf-space/skills/llm-provider-orchestration/SKILL.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: llm-provider-orchestration -description: Abstract LLM providers behind a common interface with streaming, blocking, fallback, and failure handling. ---- - -# llm-provider-orchestration - -## Purpose - -Prevent application logic from becoming dependent on one model provider and keep inference resilient. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Define a provider-neutral chat and streaming contract. -- Register available providers and their capabilities. -- Attempt the preferred inference path. -- Detect timeout, provider, or streaming failures. -- Fall back according to an explicit priority ladder. -- Return provider/fallback metadata for observability. -- Keep prompts and application semantics independent of provider-specific APIs. - -## Guardrails - -- Never silently change providers without recording that fallback occurred. -- Keep provider credentials out of application code. -- Set explicit timeouts and token limits. -- Do not make provider-specific response formats leak into core application logic. - -## Expected outputs - -- provider-neutral inference client -- fallback ladder -- streaming/blocking interface -- inference telemetry - -## Example - -A streaming request fails at the primary provider. The orchestrator retries through the approved blocking or secondary-provider path and records that fallback was used. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/skills/ml-observability/SKILL.md b/hf-space/skills/ml-observability/SKILL.md deleted file mode 100644 index aca3232dddf7e3147b1a6e26493b8360dcdc8911..0000000000000000000000000000000000000000 --- a/hf-space/skills/ml-observability/SKILL.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -name: ml-observability -description: Monitor deployed ML systems through quality, drift, agreement, sampling, and operational metrics. ---- - -# ml-observability - -## Purpose - -Detect degradation and understand whether an ML system remains reliable as data and operating conditions change. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Define model-quality metrics appropriate to the task. -- Track metrics across time and relevant cohorts. -- Compare predictions with trusted labels or reference sets. -- Monitor input/class distribution and sampling changes. -- Track drift and agreement metrics. -- Maintain a stable evaluation or golden set where applicable. -- Surface anomalies and data-quality gaps as first-class signals. - -## Guardrails - -- Do not treat a single metric as model health. -- Distinguish model degradation from label/data-pipeline changes. -- Record denominators and sample sizes. -- Flag insufficient history rather than manufacturing a trend. - -## Expected outputs - -- model health dashboard -- quality trends -- drift indicators -- agreement metrics -- evaluation-set monitoring - -## Example - -A model's headline metric is stable, but class mix and sampling distributions have shifted. The observability layer surfaces both signals so the team does not mistake metric stability for unchanged operating conditions. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/skills/natural-language-ui-controller/SKILL.md b/hf-space/skills/natural-language-ui-controller/SKILL.md deleted file mode 100644 index 1f2e6dbf803172f1ee4498493854f13e58adbe4a..0000000000000000000000000000000000000000 --- a/hf-space/skills/natural-language-ui-controller/SKILL.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -name: natural-language-ui-controller -description: Convert natural-language requests into validated UI navigation, filtering, and interaction commands. ---- - -# natural-language-ui-controller - -## Purpose - -Make an analytical application controllable through natural language without coupling the LLM directly to frontend implementation details. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Classify the request as navigation/filtering, analysis, or a mixed request. -- Emit a constrained action object containing only supported tabs, filters, and actions. -- Validate the action against current dimension values and UI capabilities. -- Apply the action to application state. -- Refresh the affected data views. -- Expose the resulting state as a shareable URL when possible. - -## Guardrails - -- Use an explicit action schema. -- Reject unknown dimension values instead of guessing. -- Do not allow arbitrary JavaScript or UI commands from the model. -- Keep navigation and analytical answering as separate modes. -- Preserve existing filters only when the action contract explicitly permits it. - -## Expected outputs - -- validated UI action -- updated UI state -- optional shareable deep link - -## Example - -User: 'Show failed cases in the last eight weeks.' The controller selects the appropriate view and applies a validated status and time-window filter. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/skills/schema-aware-analytics/SKILL.md b/hf-space/skills/schema-aware-analytics/SKILL.md deleted file mode 100644 index 55071817da0d618e18a82561d40ecdf375dded37..0000000000000000000000000000000000000000 --- a/hf-space/skills/schema-aware-analytics/SKILL.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -name: schema-aware-analytics -description: Discover, validate, and use data schemas dynamically so analytical applications can adapt to changing datasets. ---- - -# schema-aware-analytics - -## Purpose - -Separate analytical semantics from physical database column names and make the data layer resilient to schema variation. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Discover available tables, columns, types, and relevant metadata. -- Map business concepts to physical fields using explicit mappings or controlled heuristics. -- Validate required fields before executing an analytical request. -- Expose only approved fields and dimensions to the query layer and agent. -- Generate parameterized queries from validated schema information. -- Report missing or ambiguous mappings rather than silently guessing. - -## Guardrails - -- Never infer a field mapping and treat it as certain without validation. -- Keep identifiers separate from user-provided query values. -- Make schema assumptions observable. -- Fail clearly when required data is unavailable. - -## Expected outputs - -- validated schema map -- available dimensions/measures -- query-ready field contract -- data-gap report - -## Example - -A dataset changes its physical column name for a timestamp field. The schema layer detects the new field, validates its type, updates the semantic mapping, and keeps the analytical layer unchanged. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/skills/streaming-llm-interface/SKILL.md b/hf-space/skills/streaming-llm-interface/SKILL.md deleted file mode 100644 index 12f8cce4b8feec11f32af72af03bc9e6ce48a0c9..0000000000000000000000000000000000000000 --- a/hf-space/skills/streaming-llm-interface/SKILL.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -name: streaming-llm-interface -description: Build responsive LLM interfaces using incremental server-sent responses and progressive rendering. ---- - -# streaming-llm-interface - -## Purpose - -Reduce perceived latency and expose model progress while preserving a structured final response. - -## When to use - -Use this skill when the task requires the capability described above. Keep domain-specific terminology out of the implementation unless the target system explicitly requires it. - -## Workflow - -- Open an SSE or equivalent streaming channel. -- Render valid partial text as it arrives. -- Keep transient stream state separate from the final structured response. -- Detect completion, timeout, cancellation, and malformed events. -- Render the final answer and associated actions once available. -- Provide a fallback path when streaming fails. - -## Guardrails - -- Never assume every streamed event is user-displayable text. -- Handle disconnects and cancellation. -- Do not lose the final structured action when the text stream is incomplete. -- Make thinking/loading/error states explicit. - -## Expected outputs - -- streaming endpoint -- incremental UI renderer -- final response contract -- fallback behavior - -## Example - -The user sees the answer progressively while the backend continues generating the final structured result and action metadata. - -## Implementation principles - -- Prefer boring, explicit interfaces over implicit agent behavior. -- Keep domain semantics configurable rather than embedded in the skill. -- Validate model-generated structured data before execution. -- Make data provenance, uncertainty, and failure states observable. -- Separate retrieval, reasoning, execution, and presentation. diff --git a/hf-space/space_app.py b/hf-space/space_app.py deleted file mode 100644 index c8fc1647c8fd6548b1535c5c5c4f7298b7251d59..0000000000000000000000000000000000000000 --- a/hf-space/space_app.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Hugging Face Space entrypoint for Sensorflow Studio. - -Wraps the existing FastAPI app from app_backend.py WITHOUT modifying it, and -serves the built React frontend (dist/) at /dashboard. - -app_backend.py mounts the legacy static UI as a catch-all at "/" during module -import, so by the time this module runs, that mount is already the last route. -A mount added via app.mount() here would land *after* the catch-all and never -match; instead the /dashboard mount is inserted just before it in the route -table. - -Run with: uvicorn space_app:app --host 0.0.0.0 --port 7860 -""" -from pathlib import Path - -from starlette.routing import Mount -from starlette.staticfiles import StaticFiles - -from app_backend import app - -_dist_dir = Path(__file__).parent / "dist" - -if _dist_dir.exists(): - _dashboard = Mount( - "/dashboard", - app=StaticFiles(directory=str(_dist_dir), html=True), - name="dashboard", - ) - _routes = app.router.routes - _root_idx = next( - ( - i - for i, r in enumerate(_routes) - if isinstance(r, Mount) and r.path in ("", "/") - ), - len(_routes), - ) - _routes.insert(_root_idx, _dashboard) diff --git a/hf-space/src/App.tsx b/hf-space/src/App.tsx deleted file mode 100644 index 4d01c367717f903ff4e4179f872f63bb4f242b9e..0000000000000000000000000000000000000000 --- a/hf-space/src/App.tsx +++ /dev/null @@ -1,295 +0,0 @@ -import { useCallback, useEffect, useMemo, useState } from 'react'; -import AppBar from '@mui/material/AppBar'; -import Box from '@mui/material/Box'; -import Chip from '@mui/material/Chip'; -import Drawer from '@mui/material/Drawer'; -import List from '@mui/material/List'; -import ListItemButton from '@mui/material/ListItemButton'; -import ListItemIcon from '@mui/material/ListItemIcon'; -import ListItemText from '@mui/material/ListItemText'; -import ListSubheader from '@mui/material/ListSubheader'; -import Toolbar from '@mui/material/Toolbar'; -import Typography from '@mui/material/Typography'; -import { - Gauge, - LayoutDashboard, - Database, - Tag, - Radar, - ShieldCheck, - TrendingDown, - Filter, - UserCheck, - Rocket, - Boxes, - ListChecks, - ScrollText, - Workflow, - Map as MapIcon, - AppWindow, -} from 'lucide-react'; -import { LabelEvalContext, type PageId, ALL_PAGE_IDS } from './context/LabelEvalContext'; -import { getOverview, useStream } from './services/labeleval'; -import OverviewPage from './pages/OverviewPage'; -import DatasetsPage from './pages/DatasetsPage'; -import LabelGenerationPage from './pages/LabelGenerationPage'; -import RareEventDashboard from './pages/RareEventDashboard'; -import QualityEnginePage from './pages/QualityEnginePage'; -import RegressionPage from './pages/RegressionPage'; -import TriagePage from './pages/TriagePage'; -import HumanReviewPage from './pages/HumanReviewPage'; -import TrainingPage from './pages/TrainingPage'; -import ModelsPage from './pages/ModelsPage'; -import EvaluationPage from './pages/EvaluationPage'; -import AuditPage from './pages/AuditPage'; -import PipelineArchitecturePage from './pages/PipelineArchitecturePage'; -import SSAMSafetyDashboard from './pages/SSAMSafetyDashboard'; -import LegacyStudioPage from './pages/LegacyStudioPage'; -import CommandCenterPage from './pages/CommandCenterPage'; - -const DRAWER_WIDTH = 230; - -interface NavItem { - id: PageId; - label: string; - icon: React.ReactNode; -} - -const PLATFORM_NAV: NavItem[] = [ - { id: 'command', label: 'Command Center', icon: }, - { id: 'overview', label: 'Overview', icon: }, - { id: 'datasets', label: 'Datasets', icon: }, - { id: 'label-generation', label: 'Label Generation', icon: }, - { id: 'rare-events', label: 'Rare Events', icon: }, - { id: 'quality', label: 'Quality Engine', icon: }, - { id: 'regression', label: 'Regression', icon: }, - { id: 'triage', label: 'Triage', icon: }, - { id: 'review', label: 'Human Review', icon: }, - { id: 'training', label: 'Training', icon: }, - { id: 'models', label: 'Models', icon: }, - { id: 'evaluation', label: 'Evaluation', icon: }, - { id: 'audit', label: 'Audit', icon: }, - { id: 'pipeline', label: 'Pipeline Architecture', icon: }, -]; - -const LEGACY_NAV: NavItem[] = [ - { id: 'ssam', label: 'SSAM Safety', icon: }, - { id: 'legacy', label: 'Legacy Studio', icon: }, -]; - -const PAGE_TITLES: Record = { - command: 'Evaluation Command Center', - overview: 'Overview', - datasets: 'Datasets', - 'label-generation': 'Label Generation', - 'rare-events': 'Rare Event Detection', - quality: 'Quality Engine', - regression: 'Regression Tracking', - triage: 'Automated Triage', - review: 'Human Review (HITL)', - training: 'Training Flywheel', - models: 'Models', - evaluation: 'Evaluation Records', - audit: 'Audit & Process Units', - pipeline: 'Pipeline Architecture', - ssam: 'SSAM Safety Dashboard', - legacy: 'Legacy Studio', -}; - -function parseHash(): { page: PageId; entityId: string | null } { - const raw = window.location.hash.replace(/^#\/?/, ''); - const [pageRaw, entityRaw] = raw.split('/'); - const page = (ALL_PAGE_IDS as string[]).includes(pageRaw ?? '') ? (pageRaw as PageId) : 'command'; - return { page, entityId: entityRaw ? decodeURIComponent(entityRaw) : null }; -} - -export default function App() { - const initial = useMemo(parseHash, []); - const [page, setPage] = useState(initial.page); - const [entityId, setEntityId] = useState(initial.entityId); - const [activeDatasetId, setActiveDatasetId] = useState(null); - const stream = useStream(); - - const navigate = useCallback((nextPage: PageId, nextEntityId?: string | null) => { - setPage(nextPage); - setEntityId(nextEntityId ?? null); - window.location.hash = nextEntityId ? `${nextPage}/${encodeURIComponent(nextEntityId)}` : nextPage; - }, []); - - useEffect(() => { - const onHashChange = () => { - const parsed = parseHash(); - setPage(parsed.page); - setEntityId(parsed.entityId); - }; - window.addEventListener('hashchange', onHashChange); - return () => window.removeEventListener('hashchange', onHashChange); - }, []); - - // Adopt the backend's active dataset once on startup so pages have context. - useEffect(() => { - getOverview() - .then((o) => { - if (o.active_dataset) { - setActiveDatasetId((current) => current ?? o.active_dataset); - } - }) - .catch(() => undefined); - }, []); - - const reviewCount = stream?.pipeline.review_queue_count ?? 0; - const alertsCount = stream?.alerts_count ?? 0; - - const badgeFor = (id: PageId): number => { - if (id === 'review') return reviewCount; - if (id === 'overview') return alertsCount; - return 0; - }; - - const contextValue = useMemo( - () => ({ activeDatasetId, setActiveDatasetId, stream, page, entityId, navigate }), - [activeDatasetId, stream, page, entityId, navigate] - ); - - const renderNavItem = (item: NavItem) => { - const badge = badgeFor(item.id); - return ( - navigate(item.id)} - sx={{ - borderRadius: 1, - mx: 0.75, - my: 0.1, - py: 0.6, - '&.Mui-selected': { bgcolor: 'rgba(79,195,247,0.12)', color: '#4fc3f7' }, - }} - > - - {item.icon} - - - {badge > 0 ? ( - 999 ? '999+' : badge.toLocaleString()} - sx={{ - height: 18, - fontSize: 10, - fontWeight: 700, - bgcolor: item.id === 'review' ? '#e65100' : '#b71c1c', - color: '#fff', - }} - /> - ) : null} - - ); - }; - - const fullBleed = page === 'ssam' || page === 'legacy'; - - return ( - - - t.zIndex.drawer + 1, bgcolor: '#12171d', backgroundImage: 'none' }}> - - - Sensorflow Studio — L4 Perception Label Evaluation - - {stream?.pipeline.running ? ( - - ) : null} - {activeDatasetId ? ( - - ) : null} - - - - - - - - PLATFORM - - } - > - {PLATFORM_NAV.map(renderNavItem)} - - - LEGACY / EXISTING - - } - > - {LEGACY_NAV.map(renderNavItem)} - - - - - - - {fullBleed ? ( - - {page === 'ssam' ? : } - - ) : ( - - - {PAGE_TITLES[page]} - - {page === 'command' ? : null} - {page === 'overview' ? : null} - {page === 'datasets' ? : null} - {page === 'label-generation' ? : null} - {page === 'rare-events' ? : null} - {page === 'quality' ? : null} - {page === 'regression' ? : null} - {page === 'triage' ? : null} - {page === 'review' ? : null} - {page === 'training' ? : null} - {page === 'models' ? : null} - {page === 'evaluation' ? : null} - {page === 'audit' ? : null} - {page === 'pipeline' ? : null} - - )} - - - - ); -} diff --git a/hf-space/src/components/DataGrid/DataGrid.tsx b/hf-space/src/components/DataGrid/DataGrid.tsx deleted file mode 100644 index d27ea9bd8bd0b007dacbdf40e864e05bb65c98da..0000000000000000000000000000000000000000 --- a/hf-space/src/components/DataGrid/DataGrid.tsx +++ /dev/null @@ -1,106 +0,0 @@ -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableContainer from '@mui/material/TableContainer'; -import TableHead from '@mui/material/TableHead'; -import TablePagination from '@mui/material/TablePagination'; -import TableRow from '@mui/material/TableRow'; -import TableSortLabel from '@mui/material/TableSortLabel'; -import Typography from '@mui/material/Typography'; -import { useFilters } from '../../context/FilterContext'; -import { SEVERITY_COLORS } from '../../types'; - -const COLUMNS: { id: string; label: string; numeric?: boolean; sortable?: boolean }[] = [ - { id: 'street_name', label: 'Intersection', sortable: true }, - { id: 'county', label: 'County', sortable: true }, - { id: 'conflict_type', label: 'Conflict' }, - { id: 'min_ttc', label: 'TTC (s)', numeric: true, sortable: true }, - { id: 'min_pet', label: 'PET (s)', numeric: true, sortable: true }, - { id: 'max_speed', label: 'Speed (m/s)', numeric: true, sortable: true }, - { id: 'severity_index', label: 'Severity', numeric: true, sortable: true }, -]; - -export default function DataGrid() { - const { data, error, page, setPage, pageSize, setPageSize, sortBy, sortDir, setSort, selected, setSelected } = - useFilters(); - - if (error) { - return ( -
    - Failed to load SSAM data: {error} -
    - ); - } - - return ( -
    - - - - - {COLUMNS.map((col) => ( - - {col.sortable ? ( - setSort(col.id)} - > - {col.label} - - ) : ( - col.label - )} - - ))} - - - - {(data?.rows ?? []).map((row) => ( - setSelected(row)} - sx={{ cursor: 'pointer' }} - > - {row.street_name} - {row.county} - {row.conflict_type} - {row.min_ttc.toFixed(1)} - {row.min_pet.toFixed(1)} - {row.max_speed.toFixed(1)} - - - {row.severity_index.toFixed(2)} {row.severity_label} - - - - ))} - {data && data.rows.length === 0 && ( - - - - No conflicts match the current filters. - - - - )} - -
    -     - setPage(p + 1)} - rowsPerPage={pageSize} - onRowsPerPageChange={(e) => { - setPageSize(parseInt(e.target.value, 10)); - setPage(1); - }} - rowsPerPageOptions={[10, 25, 50, 100]} - /> -
    - ); -} diff --git a/hf-space/src/components/FilterBar/FilterBar.tsx b/hf-space/src/components/FilterBar/FilterBar.tsx deleted file mode 100644 index 06040bd1d7784429dd0270427e97a64d3576708a..0000000000000000000000000000000000000000 --- a/hf-space/src/components/FilterBar/FilterBar.tsx +++ /dev/null @@ -1,137 +0,0 @@ -import Autocomplete from '@mui/material/Autocomplete'; -import Chip from '@mui/material/Chip'; -import IconButton from '@mui/material/IconButton'; -import Slider from '@mui/material/Slider'; -import TextField from '@mui/material/TextField'; -import Tooltip from '@mui/material/Tooltip'; -import Typography from '@mui/material/Typography'; -import { RotateCcw, Search, ShieldAlert } from 'lucide-react'; -import { useFilters } from '../../context/FilterContext'; -import { SEVERITY_COLORS, SeverityLabel } from '../../types'; - -const SEVERITY_ORDER: SeverityLabel[] = ['Critical', 'High', 'Medium', 'Low']; - -export default function FilterBar() { - const { filters, setFilters, resetFilters, data, loading } = useFilters(); - const options = data?.filter_options; - - return ( -
    -
    - -
    - - SSAM Safety Dashboard - - - California statewide surrogate safety conflicts - -
    -
    - - setFilters({ search: e.target.value })} - InputProps={{ startAdornment: }} - sx={{ minWidth: 220 }} - /> - - setFilters({ counties: v })} - renderInput={(params) => } - sx={{ minWidth: 200 }} - /> - - setFilters({ conflictTypes: v })} - renderInput={(params) => } - sx={{ minWidth: 190 }} - /> - - setFilters({ severityLabels: v })} - renderInput={(params) => } - sx={{ minWidth: 180 }} - /> - -
    - - Max TTC: {filters.ttcMax ?? '—'}s - - setFilters({ ttcMax: v === 1.7 ? null : (v as number) })} - sx={{ width: 110 }} - /> -
    - -
    - - Min speed: {filters.speedMin ?? '—'} m/s - - setFilters({ speedMin: v === 0 ? null : (v as number) })} - sx={{ width: 110 }} - /> -
    - -
    - {SEVERITY_ORDER.map((label) => ( - - setFilters({ - severityLabels: filters.severityLabels.includes(label) - ? filters.severityLabels.filter((l) => l !== label) - : [...filters.severityLabels, label], - }) - } - variant={filters.severityLabels.includes(label) ? 'filled' : 'outlined'} - sx={{ - borderColor: SEVERITY_COLORS[label], - color: filters.severityLabels.includes(label) ? '#111' : SEVERITY_COLORS[label], - bgcolor: filters.severityLabels.includes(label) ? SEVERITY_COLORS[label] : 'transparent', - fontWeight: 600, - }} - /> - ))} -
    - - - - - - - - -
    - ); -} diff --git a/hf-space/src/components/MapCanvas/MapCanvas.tsx b/hf-space/src/components/MapCanvas/MapCanvas.tsx deleted file mode 100644 index b33a0efdbf0278cb4f1a6cfbb00b3e6e4b83bb8d..0000000000000000000000000000000000000000 --- a/hf-space/src/components/MapCanvas/MapCanvas.tsx +++ /dev/null @@ -1,123 +0,0 @@ -import { useMemo, useState } from 'react'; -import DeckGL from '@deck.gl/react'; -import { TileLayer } from '@deck.gl/geo-layers'; -import { BitmapLayer, ScatterplotLayer } from '@deck.gl/layers'; -import CircularProgress from '@mui/material/CircularProgress'; -import { useFilters } from '../../context/FilterContext'; -import { GeoJSONFeature, SEVERITY_COLORS, SEVERITY_RGB } from '../../types'; - -const INITIAL_VIEW_STATE = { - longitude: -119.4, - latitude: 36.6, - zoom: 5.3, - pitch: 0, - bearing: 0, -}; - -interface HoverInfo { - x: number; - y: number; - feature: GeoJSONFeature; -} - -interface PickInfo { - x: number; - y: number; - object?: GeoJSONFeature; -} - -export default function MapCanvas() { - const { data, loading, selected, setSelected } = useFilters(); - const [hover, setHover] = useState(null); - - const features = data?.geojson.features ?? []; - - const layers = useMemo(() => { - const basemap = new TileLayer({ - id: 'carto-dark-basemap', - data: 'https://basemaps.cartocdn.com/dark_all/{z}/{x}/{y}.png', - minZoom: 0, - maxZoom: 19, - tileSize: 256, - renderSubLayers: (props: { - tile: { bbox: { west: number; south: number; east: number; north: number } }; - data: unknown; - }) => { - const { west, south, east, north } = props.tile.bbox; - return new BitmapLayer(props, { - data: undefined, - image: props.data, - bounds: [west, south, east, north], - }); - }, - }); - - const points = new ScatterplotLayer({ - id: 'ssam-conflicts', - data: features, - pickable: true, - stroked: true, - radiusUnits: 'pixels', - lineWidthUnits: 'pixels', - getPosition: (f: GeoJSONFeature) => f.geometry.coordinates, - getRadius: (f: GeoJSONFeature) => 4 + f.properties.severity_index * 10, - getFillColor: (f: GeoJSONFeature) => { - const [r, g, b] = SEVERITY_RGB[f.properties.severity_label] ?? [128, 128, 128]; - return [r, g, b, 200]; - }, - getLineColor: (f: GeoJSONFeature) => - selected && f.properties.id === selected.id ? [255, 255, 255, 255] : [0, 0, 0, 120], - getLineWidth: (f: GeoJSONFeature) => (selected && f.properties.id === selected.id ? 3 : 1), - onHover: (info: PickInfo) => - setHover(info.object ? { x: info.x, y: info.y, feature: info.object } : null), - onClick: (info: PickInfo) => { - const feature = info.object; - if (!feature) return; - const [lng, lat] = feature.geometry.coordinates; - setSelected({ ...feature.properties, lng, lat }); - }, - updateTriggers: { - getLineColor: [selected?.id], - getLineWidth: [selected?.id], - }, - }); - - return [basemap, points]; - }, [features, selected, setSelected]); - - return ( -
    - - isDragging ? 'grabbing' : isHovering ? 'pointer' : 'grab' - } - /> - - {hover && ( -
    - {hover.feature.properties.street_name} - {hover.feature.properties.county} County - - {hover.feature.properties.severity_label} · index {hover.feature.properties.severity_index.toFixed(2)} - - - TTC {hover.feature.properties.min_ttc.toFixed(1)}s · PET {hover.feature.properties.min_pet.toFixed(1)}s - -
    - )} - - {loading && ( -
    - -
    - )} - -
    - © OpenStreetMap · © CARTO -
    -
    - ); -} diff --git a/hf-space/src/components/SeverityDrawer/SeverityDrawer.tsx b/hf-space/src/components/SeverityDrawer/SeverityDrawer.tsx deleted file mode 100644 index b48b264faf931ada6273c2598b698259f1de7060..0000000000000000000000000000000000000000 --- a/hf-space/src/components/SeverityDrawer/SeverityDrawer.tsx +++ /dev/null @@ -1,149 +0,0 @@ -import { useEffect, useState } from 'react'; -import Alert from '@mui/material/Alert'; -import Button from '@mui/material/Button'; -import Divider from '@mui/material/Divider'; -import Drawer from '@mui/material/Drawer'; -import IconButton from '@mui/material/IconButton'; -import LinearProgress from '@mui/material/LinearProgress'; -import TextField from '@mui/material/TextField'; -import Typography from '@mui/material/Typography'; -import { X } from 'lucide-react'; -import { annotateStreet } from '../../services/api'; -import { useFilters } from '../../context/FilterContext'; -import { SEVERITY_COLORS } from '../../types'; - -export default function SeverityDrawer() { - const { selected, setSelected } = useFilters(); - const [annotation, setAnnotation] = useState(''); - const [saving, setSaving] = useState(false); - const [saveState, setSaveState] = useState<'idle' | 'saved' | 'error'>('idle'); - - useEffect(() => { - setAnnotation(selected?.manual_annotation ?? ''); - setSaveState('idle'); - }, [selected?.id]); - - const handleSave = async () => { - if (!selected) return; - setSaving(true); - try { - await annotateStreet(selected.street_name, annotation); - setSaveState('saved'); - } catch { - setSaveState('error'); - } finally { - setSaving(false); - } - }; - - const severityColor = selected ? SEVERITY_COLORS[selected.severity_label] : '#888'; - - return ( - - {selected && ( - <> -
    -
    - - {selected.street_name} - - - {selected.county} County · {selected.id} - -
    - setSelected(null)} aria-label="Close details"> - - -
    - -
    - - Severity index - -
    - {selected.severity_index.toFixed(2)} - - {selected.severity_label} - -
    - -
    - - - -
    -
    -
    Conflict type
    -
    {selected.conflict_type}
    -
    -
    -
    Min time-to-collision
    -
    {selected.min_ttc.toFixed(1)} s
    -
    -
    -
    Min post-encroachment time
    -
    {selected.min_pet.toFixed(1)} s
    -
    -
    -
    Max approach speed
    -
    {selected.max_speed.toFixed(1)} m/s
    -
    -
    -
    Location
    -
    - {selected.lat.toFixed(4)}, {selected.lng.toFixed(4)} -
    -
    -
    - - - - - Analyst annotation - - { - setAnnotation(e.target.value); - setSaveState('idle'); - }} - sx={{ mt: 1 }} - /> - - {saveState === 'saved' && ( - - Annotation saved. - - )} - {saveState === 'error' && ( - - Failed to save annotation. - - )} - - )} -     - ); -} diff --git a/hf-space/src/components/labeleval/AnomalyDetectionConfig.tsx b/hf-space/src/components/labeleval/AnomalyDetectionConfig.tsx deleted file mode 100644 index 5dbdd43388395bce15765e2fd5b1c4754269221a..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/AnomalyDetectionConfig.tsx +++ /dev/null @@ -1,108 +0,0 @@ -import Box from '@mui/material/Box'; -import Switch from '@mui/material/Switch'; -import TextField from '@mui/material/TextField'; -import Typography from '@mui/material/Typography'; -import type { AnomalyConfig } from '../../types/labeleval'; - -type Detectors = AnomalyConfig['detectors']; - -function DetectorRow({ - label, - enabled, - onToggle, - children, -}: { - label: string; - enabled: boolean; - onToggle: (enabled: boolean) => void; - children?: React.ReactNode; -}) { - return ( - - - - {label} - - onToggle(e.target.checked)} /> - - {enabled ? {children} : null} - - ); -} - -function NumField({ - label, - value, - onChange, - step, -}: { - label: string; - value: number; - onChange: (v: number) => void; - step?: number; -}) { - return ( - onChange(Number(e.target.value))} - sx={{ flex: 1, '& input': { fontSize: 13 } }} - /> - ); -} - -/** Column 2 of the rare-event configuration grid: classical anomaly detectors. */ -export default function AnomalyDetectionConfig({ - detectors, - onChange, -}: { - detectors: Detectors; - onChange: (detectors: Detectors) => void; -}) { - const set = (key: K, patch: Partial) => - onChange({ ...detectors, [key]: { ...detectors[key], ...patch } }); - - return ( - - set('knn', { enabled: v })}> - set('knn', { k: v })} /> - - set('lof', { enabled: v })}> - set('lof', { n_neighbors: v })} - /> - - set('isolation_forest', { enabled: v })} - > - set('isolation_forest', { n_estimators: v })} - /> - - set('ocsvm', { enabled: v })}> - set('ocsvm', { nu: v })} /> - - set('dbscan', { enabled: v })} - > - set('dbscan', { eps: v })} /> - set('dbscan', { min_samples: v })} - /> - - - ); -} diff --git a/hf-space/src/components/labeleval/BenchmarkResults.tsx b/hf-space/src/components/labeleval/BenchmarkResults.tsx deleted file mode 100644 index 2c247924f9d71308c62a58666bd617af57c9368f..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/BenchmarkResults.tsx +++ /dev/null @@ -1,168 +0,0 @@ -import { useMemo, useState } from 'react'; -import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import Chip from '@mui/material/Chip'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import TableSortLabel from '@mui/material/TableSortLabel'; -import Typography from '@mui/material/Typography'; -import Alert from '@mui/material/Alert'; -import CircularProgress from '@mui/material/CircularProgress'; -import { FlaskConical } from 'lucide-react'; -import type { BenchmarkResponse, BenchmarkRow } from '../../types/labeleval'; -import { runBenchmark } from '../../services/labeleval'; -import { SectionCard, fmtNum, fmtPct } from './shared'; - -type NumericKey = Exclude; -type SortKey = keyof BenchmarkRow; - -const COLUMNS: Array<{ key: SortKey; label: string; pct?: boolean }> = [ - { key: 'technique', label: 'Technique' }, - { key: 'precision', label: 'Precision', pct: true }, - { key: 'recall', label: 'Recall', pct: true }, - { key: 'rare_recall', label: 'Rare Recall', pct: true }, - { key: 'f1', label: 'F1', pct: true }, - { key: 'box_error_3d', label: '3D Box Error' }, - { key: 'id_swap_rate', label: 'ID Swap', pct: true }, - { key: 'consensus', label: 'Consensus', pct: true }, - { key: 'fp_rate', label: 'FP Rate', pct: true }, - { key: 'process_units', label: 'Process Units' }, -]; - -const HIGHLIGHTS: Array<{ key: keyof BenchmarkResponse['highlights']; label: string; color: string }> = [ - { key: 'best_rare_recall', label: 'Best rare recall', color: '#ef5350' }, - { key: 'best_safety_recall', label: 'Best safety recall', color: '#ffa726' }, - { key: 'lowest_fp_rate', label: 'Lowest FP rate', color: '#ab47bc' }, - { key: 'lowest_process_units', label: 'Lowest process units', color: '#66bb6a' }, - { key: 'lowest_tracking_error', label: 'Lowest tracking error', color: '#42a5f5' }, -]; - -export default function BenchmarkResults({ - benchmark, - activeDatasetId, - onRefresh, -}: { - benchmark: BenchmarkResponse | null; - activeDatasetId: string | null; - onRefresh: () => void; -}) { - const [sortKey, setSortKey] = useState('rare_recall'); - const [sortDir, setSortDir] = useState<'asc' | 'desc'>('desc'); - const [running, setRunning] = useState(false); - const [errorMsg, setErrorMsg] = useState(null); - - const rows = useMemo(() => { - const rows = [...(benchmark?.rows ?? [])]; - rows.sort((a, b) => { - if (sortKey === 'technique') { - return sortDir === 'asc' ? a.technique.localeCompare(b.technique) : b.technique.localeCompare(a.technique); - } - const av = a[sortKey as NumericKey]; - const bv = b[sortKey as NumericKey]; - return sortDir === 'asc' ? av - bv : bv - av; - }); - return rows; - }, [benchmark, sortKey, sortDir]); - - const run = async () => { - setRunning(true); - setErrorMsg(null); - try { - await runBenchmark(activeDatasetId); - onRefresh(); - } catch (err) { - setErrorMsg(err instanceof Error ? err.message : String(err)); - } finally { - setRunning(false); - } - }; - - const toggleSort = (key: SortKey) => { - if (key === sortKey) setSortDir((d) => (d === 'asc' ? 'desc' : 'asc')); - else { - setSortKey(key); - setSortDir(key === 'technique' ? 'asc' : 'desc'); - } - }; - - return ( - : } - disabled={running} - onClick={() => void run()} - > - Run Benchmark - - } - > - {errorMsg ? ( - setErrorMsg(null)} sx={{ mb: 1 }}> - {errorMsg} - - ) : null} - - {benchmark && rows.length > 0 ? ( - <> - - {HIGHLIGHTS.map((h) => { - const technique = benchmark.highlights[h.key]; - return technique ? ( - - ) : null; - })} - - - - - {COLUMNS.map((col) => ( - - toggleSort(col.key)} - > - {col.label} - - - ))} - - - - {rows.map((r) => ( - - {r.technique} - {COLUMNS.slice(1).map((col) => ( - - {col.pct ? fmtPct(r[col.key as NumericKey]) : fmtNum(r[col.key as NumericKey], 2)} - - ))} - - ))} - -
    - - Benchmark {benchmark.benchmark_id} · {new Date(benchmark.created_at).toLocaleString()} - - - ) : ( - - No benchmark results yet. Run the benchmark to compare detection techniques across quality, rare-event - recall, tracking and compute-cost axes. - - )} -     - ); -} diff --git a/hf-space/src/components/labeleval/CopilotDrawer.tsx b/hf-space/src/components/labeleval/CopilotDrawer.tsx deleted file mode 100644 index 660888ee5538ce65040fa99fbf6b4b0a0a5dcd88..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/CopilotDrawer.tsx +++ /dev/null @@ -1,230 +0,0 @@ -import { useEffect, useState } from 'react'; -import Drawer from '@mui/material/Drawer'; -import Box from '@mui/material/Box'; -import Typography from '@mui/material/Typography'; -import Button from '@mui/material/Button'; -import Chip from '@mui/material/Chip'; -import Divider from '@mui/material/Divider'; -import LinearProgress from '@mui/material/LinearProgress'; -import CircularProgress from '@mui/material/CircularProgress'; -import Alert from '@mui/material/Alert'; -import IconButton from '@mui/material/IconButton'; -import { Sparkles, X } from 'lucide-react'; -import type { CopilotExplainRequest, CopilotExplainResponse } from '../../types/labeleval'; -import { copilotExplain } from '../../services/labeleval'; - -export default function CopilotDrawer({ - open, - onClose, - request, -}: { - open: boolean; - onClose: () => void; - request: CopilotExplainRequest | null; -}) { - const [result, setResult] = useState(null); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); - - useEffect(() => { - // Reset when a new context arrives. - setResult(null); - setError(null); - }, [request]); - - const analyze = async () => { - if (!request) return; - setLoading(true); - setError(null); - try { - const res = await copilotExplain(request); - setResult(res); - } catch (err) { - // A 503 may still carry an offline-deterministic analysis in its body. - const message = err instanceof Error ? err.message : String(err); - const jsonStart = message.indexOf('{'); - if (jsonStart >= 0) { - try { - const body = JSON.parse(message.slice(jsonStart)) as Partial; - if (typeof body.analysis === 'string' && body.analysis) { - setResult({ - status: body.status ?? 'offline', - provider: body.provider ?? 'offline_deterministic', - analysis: body.analysis, - structured: body.structured ?? null, - }); - setLoading(false); - return; - } - } catch { - /* body was not JSON */ - } - } - setError(`Copilot unavailable: ${message}`); - } finally { - setLoading(false); - } - }; - - const structured = result?.structured ?? null; - - return ( - - - - - - Copilot - - - - - - - - Advisory only — the Copilot never changes metrics or overrides gates. - - - {request ? ( - - - CONTEXT - - - - {request.annotation_id ? : null} - {request.event_id ? : null} - {request.model_version ? : null} - - - ) : ( - - No context selected. - - )} - - - - {error ? ( - - {error} The Copilot service may be offline — evidence and gate results above remain authoritative. - - ) : null} - - {result ? ( - - - Provider: {result.provider} · Status: {result.status} - - - {structured ? ( - <> - - - FAILURE CLASSIFICATION - - - {structured.failure_classification} - - - - - OBSERVED EVIDENCE - -
      - {structured.observed_evidence.map((e, i) => ( -
    • - {e} -
      • - ))} -
    -     - - - LIKELY CAUSE - - {structured.likely_cause} - - {structured.contributing_factors.length > 0 ? ( - - - CONTRIBUTING FACTORS - -
      - {structured.contributing_factors.map((f, i) => ( -
    • - {f} -
      • - ))} -
    -     - ) : null} - - - Hypothesis — not a conclusion - - {structured.hypothesis} - - {structured.recommended_investigation.length > 0 ? ( - - - RECOMMENDED INVESTIGATION - -
      - {structured.recommended_investigation.map((r, i) => ( -
    • - {r} -
      • - ))} -
    -     - ) : null} - - - CONFIDENCE {(structured.confidence * 100).toFixed(0)}% - - - - - - ) : null} - - {result.analysis ? ( - - - ANALYSIS - - - {result.analysis} - - - ) : null} -     - ) : null} -     -     - ); -} diff --git a/hf-space/src/components/labeleval/DatasetSelector.tsx b/hf-space/src/components/labeleval/DatasetSelector.tsx deleted file mode 100644 index 2c6ad65eeb5cd86e089bb20617b198c1b2178f6a..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/DatasetSelector.tsx +++ /dev/null @@ -1,296 +0,0 @@ -import { useState } from 'react'; -import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import Chip from '@mui/material/Chip'; -import Dialog from '@mui/material/Dialog'; -import DialogTitle from '@mui/material/DialogTitle'; -import DialogContent from '@mui/material/DialogContent'; -import DialogActions from '@mui/material/DialogActions'; -import TextField from '@mui/material/TextField'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import Alert from '@mui/material/Alert'; -import CircularProgress from '@mui/material/CircularProgress'; -import { DatabaseZap, Play, ShieldQuestion } from 'lucide-react'; -import type { DatasetSummary } from '../../types/labeleval'; -import { - generateDataset, - precheckDataset, - runPipeline, - type PrecheckResponse, -} from '../../services/labeleval'; -import { SectionCard, StatusChip, GateLineList, fmtInt, fmtPct, EmptyState } from './shared'; - -function lineageChips(d: DatasetSummary) { - const chips: string[] = []; - if (d.lineage.generated_from_model) chips.push(`from ${d.lineage.generated_from_model}`); - if (d.lineage.corrected_by_review_batch) chips.push(`corrected by ${d.lineage.corrected_by_review_batch}`); - if (d.lineage.validated_by_policy) chips.push(`policy ${d.lineage.validated_by_policy}`); - if (d.lineage.parent_dataset) chips.push(`parent ${d.lineage.parent_dataset}`); - return chips; -} - -export default function DatasetSelector({ - datasets, - selectedId, - onSelect, - onChanged, - verificationRates, -}: { - datasets: DatasetSummary[]; - selectedId: string | null; - onSelect: (id: string) => void; - /** Called after generate / run so the parent can refresh. */ - onChanged: () => void; - /** Optional map dataset_id -> verification rate (0..1). */ - verificationRates?: Record; -}) { - const [genOpen, setGenOpen] = useState(false); - const [genForm, setGenForm] = useState({ name: '', num_sequences: 8, frames_per_sequence: 25, seed: 42 }); - const [busy, setBusy] = useState(null); // 'generate' | `precheck:${id}` | `run:${id}` - const [precheck, setPrecheck] = useState<{ datasetId: string; result: PrecheckResponse } | null>(null); - const [notice, setNotice] = useState(null); - const [errorMsg, setErrorMsg] = useState(null); - - const doGenerate = async () => { - setBusy('generate'); - setErrorMsg(null); - try { - const ds = await generateDataset({ - ...(genForm.name ? { name: genForm.name } : {}), - num_sequences: genForm.num_sequences, - frames_per_sequence: genForm.frames_per_sequence, - seed: genForm.seed, - }); - setGenOpen(false); - setNotice(`Generated dataset ${ds.name} (${ds.dataset_id})`); - onSelect(ds.dataset_id); - onChanged(); - } catch (err) { - setErrorMsg(err instanceof Error ? err.message : String(err)); - } finally { - setBusy(null); - } - }; - - const doPrecheck = async (id: string) => { - setBusy(`precheck:${id}`); - setErrorMsg(null); - try { - const result = await precheckDataset(id); - setPrecheck({ datasetId: id, result }); - } catch (err) { - setErrorMsg(err instanceof Error ? err.message : String(err)); - } finally { - setBusy(null); - } - }; - - const doRun = async (id: string) => { - setBusy(`run:${id}`); - setErrorMsg(null); - try { - const res = await runPipeline(id); - setNotice(`Pipeline run ${res.run_id} started (${res.status}). Watch progress on the Overview / Pipeline pages.`); - onChanged(); - } catch (err) { - setErrorMsg(err instanceof Error ? err.message : String(err)); - } finally { - setBusy(null); - } - }; - - return ( - } - onClick={() => setGenOpen(true)} - > - Generate Synthetic Dataset - - } - > - {notice ? ( - setNotice(null)} sx={{ mb: 1 }}> - {notice} - - ) : null} - {errorMsg ? ( - setErrorMsg(null)} sx={{ mb: 1 }}> - {errorMsg} - - ) : null} - - {datasets.length === 0 ? ( - } onClick={() => setGenOpen(true)}> - Generate Synthetic Dataset - - } - /> - ) : ( - - - - Name - Version - Frames - Annotations - Ground truth - Verification - Lineage - Status - Actions - - - - {datasets.map((d) => { - const rate = verificationRates?.[d.dataset_id]; - return ( - onSelect(d.dataset_id)} - sx={{ cursor: 'pointer' }} - > - {d.name} - {d.version} - {fmtInt(d.num_frames)} - {fmtInt(d.num_annotations)} - - {d.gt_availability.has_reference && d.gt_availability.gt_type ? ( - - ) : ( - - )} - - - {rate !== undefined ? fmtPct(rate) : '—'} - - - - {lineageChips(d).map((c) => ( - - ))} - - - - - - e.stopPropagation()}> - - - - - - - ); - })} - -
    - )} - - {/* Generate dialog */} - setGenOpen(false)}> - Generate Synthetic Dataset - - setGenForm({ ...genForm, name: e.target.value })} - /> - setGenForm({ ...genForm, num_sequences: Number(e.target.value) })} - /> - setGenForm({ ...genForm, frames_per_sequence: Number(e.target.value) })} - /> - setGenForm({ ...genForm, seed: Number(e.target.value) })} - /> - - - - - - - - {/* Precheck results dialog */} - setPrecheck(null)} maxWidth="sm" fullWidth> - Precheck — {precheck?.datasetId} - - {precheck ? ( - <> - - {precheck.result.status} — {precheck.result.message} - - - - ) : null} - - - - - -     - ); -} diff --git a/hf-space/src/components/labeleval/EvidencePanel.tsx b/hf-space/src/components/labeleval/EvidencePanel.tsx deleted file mode 100644 index b23a6bd4fc3220bbb679261e0af27aa6d174547f..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/EvidencePanel.tsx +++ /dev/null @@ -1,249 +0,0 @@ -import Drawer from '@mui/material/Drawer'; -import Box from '@mui/material/Box'; -import Typography from '@mui/material/Typography'; -import Button from '@mui/material/Button'; -import Chip from '@mui/material/Chip'; -import Divider from '@mui/material/Divider'; -import IconButton from '@mui/material/IconButton'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableRow from '@mui/material/TableRow'; -import { Sparkles, X } from 'lucide-react'; -import type { CopilotExplainRequest, EvaluationRecord, RareEvent } from '../../types/labeleval'; -import { StatusChip, GateLineList, fmtNum, fmtPct, HBar } from './shared'; - -function MetricRow({ label, value }: { label: string; value: string }) { - return ( - - {label} - - {value} - - - ); -} - -export function evidenceCopilotRequest(record: EvaluationRecord): CopilotExplainRequest { - const primary = record.decision?.primary_failure_reason ?? null; - let contextType: CopilotExplainRequest['context_type'] = 'general'; - if (record.anomaly.is_anomaly) contextType = 'anomaly'; - if (primary === 'GRADER_DISAGREEMENT') contextType = 'disagreement'; - if (primary === 'MODEL_REGRESSION') contextType = 'regression'; - return { - context_type: contextType, - annotation_id: record.annotation_id, - model_version: record.model_version, - }; -} - -export default function EvidencePanel({ - open, - onClose, - record, - event, - onAskCopilot, -}: { - open: boolean; - onClose: () => void; - record?: EvaluationRecord | null; - event?: RareEvent | null; - onAskCopilot: (request: CopilotExplainRequest) => void; -}) { - return ( - - - - Evidence - - - - - - {record ? ( - <> - - - - - {record.decision ? : null} - - - Frame {record.frame_id} · Dataset {record.dataset_id} · Detection confidence{' '} - {fmtNum(record.detection.confidence)} - {record.ground_truth_type ? ` · GT: ${record.ground_truth_type}` : ' · No ground truth reference'} - - - - - Geometry - - - - - - - - - - - -
    - - - - Anomaly ensemble — {record.anomaly.ensemble_strategy.replace(/_/g, ' ')} · threshold{' '} - {fmtNum(record.anomaly.decision_threshold)} - - - - Score {fmtNum(record.anomaly.score)} - - - - {Object.entries(record.anomaly.detector_scores).map(([name, score]) => ( - - ))} - - - - Grading ({record.grading.grader_count} graders) - - - - - - - - -
    - {record.grading.disagreement_types.length > 0 ? ( - - {record.grading.disagreement_types.map((d) => ( - - ))} - - ) : null} - - - - Tracking - - - ID switch: {record.tracking.id_switch ? 'yes' : 'no'} · Fragmentation:{' '} - {record.tracking.fragmentation ? 'yes' : 'no'} · Track quality: {fmtNum(record.tracking.track_quality)} - - - - - Quality gates {record.validation.passed ? '— PASSED' : '— FAILED'} - - - - {record.decision ? ( - <> - - - Triage decision — policy {record.decision.policy_id} - - {record.decision.primary_failure_reason ? ( - - Primary failure: {record.decision.primary_failure_reason.replace(/_/g, ' ')} - - ) : null} - - - ) : null} - - {record.injected_errors.length > 0 ? ( - <> - - - Synthetic injected defects (demo transparency): {record.injected_errors.join(', ')} - - - ) : null} - - - - ) : null} - - {event ? ( - <> - - - - - {event.verified ? : null} - - {event.description} - - - - - - - -
    - - Evidence frames ({event.evidence_frames.length}) - - - {event.evidence_frames.map((f) => ( - - ))} - - {Object.keys(event.sensor_evidence).length > 0 ? ( - <> - - Sensor evidence - - - - {Object.entries(event.sensor_evidence).map(([sensor, note]) => ( - - ))} - -
    - - ) : null} - - - ) : null} - - {!record && !event ? ( - - No evidence selected. - - ) : null} -     -     - ); -} diff --git a/hf-space/src/components/labeleval/GraderDisagreement.tsx b/hf-space/src/components/labeleval/GraderDisagreement.tsx deleted file mode 100644 index 7f7c8f302f331b677dddd144bfd207caf74dfef0..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/GraderDisagreement.tsx +++ /dev/null @@ -1,87 +0,0 @@ -import Box from '@mui/material/Box'; -import Chip from '@mui/material/Chip'; -import Typography from '@mui/material/Typography'; -import type { QualityMetrics } from '../../types/labeleval'; -import { SectionCard, HBar, fmtPct, pctFraction } from './shared'; - -function ConsensusGauge({ value }: { value: number | null }) { - const frac = pctFraction(value); - const size = 140; - const stroke = 12; - const r = (size - stroke) / 2; - const circumference = Math.PI * r; // half circle - const color = frac >= 0.8 ? '#66bb6a' : frac >= 0.6 ? '#ffa726' : '#ef5350'; - return ( - - - - - - {value === null ? '—' : fmtPct(value)} - - - - Canonical grader_consensus_score - - - ); -} - -export default function GraderDisagreement({ - metrics, - graderCount = 3, - disagreementTypes = [], -}: { - metrics: QualityMetrics | null; - graderCount?: number; - disagreementTypes?: string[]; -}) { - const consensus = metrics?.global.grader_consensus ?? null; - const frac = pctFraction(consensus); - return ( - - - - - - AGREEMENT DIMENSIONS (dataset aggregate) - - - - - - - - Aggregate approximated by the consensus score; the per-annotation class/spatial/temporal breakdown is shown - in each evidence panel. - - {disagreementTypes.length > 0 ? ( - - {disagreementTypes.map((d) => ( - - ))} - - ) : null} - - - - Agreement statistic: Cohen's κ for 2 graders, Fleiss' κ for a fixed panel of 3+, Krippendorff's α - when graders have missing ratings. Low consensus is evidence for the Quality Gate — it never directly rejects a - label. - - - ); -} diff --git a/hf-space/src/components/labeleval/GroundTruthComparison.tsx b/hf-space/src/components/labeleval/GroundTruthComparison.tsx deleted file mode 100644 index f5fbf6bf595f6e17a21b52c5b3dbccd10e251430..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/GroundTruthComparison.tsx +++ /dev/null @@ -1,112 +0,0 @@ -import Box from '@mui/material/Box'; -import Chip from '@mui/material/Chip'; -import Alert from '@mui/material/Alert'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import type { ClassMetrics, QualityMetrics, ScenarioMetrics } from '../../types/labeleval'; -import { SectionCard, MetricCard, fmtPct, fmtInt } from './shared'; - -function PrTable({ title, rows }: { title: string; rows: Array }) { - return ( - - - {title} - - {rows.length === 0 ? ( - - No data yet. - - ) : ( - - - - {'class_name' in (rows[0] ?? {}) ? 'Class' : 'Scenario'} - Precision - Recall - F1 - Support - - - - {rows.map((r) => { - const name = 'class_name' in r ? r.class_name : r.scenario; - return ( - - {name.replace(/_/g, ' ')} - - {fmtPct(r.precision)} - - - {fmtPct(r.recall)} - - - {fmtPct(r.f1)} - - - {fmtInt(r.support)} - - - ); - })} - -
    - )} -     - ); -} - -export default function GroundTruthComparison({ metrics }: { metrics: QualityMetrics | null }) { - const gtType = metrics?.gt_type ?? null; - return ( - - {!metrics ? ( - - No quality metrics yet — run the evaluation pipeline. - - ) : ( - <> - - - {gtType ? ( - - ) : null} - - - - - {gtType === 'PSEUDO_GROUND_TRUTH' ? ( - - Pseudo-GT comparison, not gold-standard evaluation — reference labels were themselves machine-generated. - Treat precision/recall below as directional signals, not certified quality figures. - - ) : null} - - - - - - - )} - - ); -} diff --git a/hf-space/src/components/labeleval/HITLReview.tsx b/hf-space/src/components/labeleval/HITLReview.tsx deleted file mode 100644 index 46eebc6bf1282bbd1eeba57b7b0918f5132aaed6..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/HITLReview.tsx +++ /dev/null @@ -1,417 +0,0 @@ -import Box from '@mui/material/Box'; -import Chip from '@mui/material/Chip'; -import List from '@mui/material/List'; -import ListItemButton from '@mui/material/ListItemButton'; -import Typography from '@mui/material/Typography'; -import type { Annotation, FrameSummary, ReviewTask } from '../../types/labeleval'; -import { SectionCard, StatusChip, GateLineList, LoadingBox, MetricCard, fmtNum, fmtPct } from './shared'; - -// ---------------------------------------------------------------- geometry helpers - -type Pt = [number, number]; - -/** Rotated BEV rectangle corners for bbox_3d = [x,y,z,l,w,h,yaw]. */ -function boxCornersBEV(bbox: readonly number[]): Pt[] { - const [x, y, , l, w, , yaw] = bbox as [number, number, number, number, number, number, number]; - const cos = Math.cos(yaw); - const sin = Math.sin(yaw); - const dl = l / 2; - const dw = w / 2; - const local: Pt[] = [ - [dl, dw], - [dl, -dw], - [-dl, -dw], - [-dl, dw], - ]; - return local.map(([lx, ly]) => [x + lx * cos - ly * sin, y + lx * sin + ly * cos]); -} - -interface BevTransform { - toSvg: (p: Pt) => Pt; -} - -function makeBevTransform(frame: FrameSummary, width: number, height: number, pad = 14): BevTransform { - const xs: number[] = []; - const ys: number[] = []; - for (const [px, py] of frame.lidar_points_bev) { - xs.push(px); - ys.push(py); - } - for (const a of frame.annotations) { - if (a.bbox_3d) { - xs.push(a.bbox_3d[0]); - ys.push(a.bbox_3d[1]); - } - } - for (const g of frame.gt_boxes) { - xs.push(g.bbox_3d[0]); - ys.push(g.bbox_3d[1]); - } - xs.push(frame.ego_pose.x); - ys.push(frame.ego_pose.y); - const minX = xs.length ? Math.min(...xs) : -50; - const maxX = xs.length ? Math.max(...xs) : 50; - const minY = ys.length ? Math.min(...ys) : -50; - const maxY = ys.length ? Math.max(...ys) : 50; - const spanX = maxX - minX || 1; - const spanY = maxY - minY || 1; - const scale = Math.min((width - 2 * pad) / spanX, (height - 2 * pad) / spanY); - const cx = (minX + maxX) / 2; - const cy = (minY + maxY) / 2; - return { - toSvg: ([px, py]) => [width / 2 + (px - cx) * scale, height / 2 - (py - cy) * scale], - }; -} - -function polygonPoints(corners: Pt[], t: BevTransform): string { - return corners.map((c) => t.toSvg(c).map((v) => v.toFixed(1)).join(',')).join(' '); -} - -// ---------------------------------------------------------------- individual views - -const VIEW_W = 440; -const VIEW_H = 250; - -function ViewFrame({ title, children }: { title: string; children: React.ReactNode }) { - return ( - - - {title} - - {children} - - ); -} - -function CameraView({ frame, highlightId }: { frame: FrameSummary; highlightId: string }) { - const sx = VIEW_W / (frame.camera.width || 1); - const sy = VIEW_H / (frame.camera.height || 1); - return ( - - - - - - - - - - - {/* horizon + lane hints for the synthetic camera image */} - - - - {frame.annotations.map((a) => { - if (!a.bbox_2d) return null; - const [bx, by, bw, bh] = a.bbox_2d; - const highlighted = a.annotation_id === highlightId; - return ( - - - - {a.class_name} {(a.confidence * 100).toFixed(0)}% - - - ); - })} - - ); -} - -function LidarView({ frame, highlightId }: { frame: FrameSummary; highlightId: string }) { - const t = makeBevTransform(frame, VIEW_W, VIEW_H); - return ( - - {frame.lidar_points_bev.map((p, i) => { - const [x, y] = t.toSvg([p[0], p[1]]); - return ; - })} - {frame.gt_boxes.map((g) => ( - - ))} - {frame.annotations.map((a) => { - if (!a.bbox_3d) return null; - const highlighted = a.annotation_id === highlightId; - return ( - - ); - })} - - ); -} - -function BirdsEyeView({ - frame, - prevFrame, - nextFrame, - highlightId, -}: { - frame: FrameSummary; - prevFrame: FrameSummary | null; - nextFrame: FrameSummary | null; - highlightId: string; -}) { - const t = makeBevTransform(frame, VIEW_W, VIEW_H); - const ego = t.toSvg([frame.ego_pose.x, frame.ego_pose.y]); - - // Track trails: connect centers of same track_id across prev/current/next. - const trails: Array<{ trackId: string; pts: Pt[]; highlighted: boolean }> = []; - for (const a of frame.annotations) { - if (!a.track_id || !a.bbox_3d) continue; - const pts: Pt[] = []; - const prevMatch = prevFrame?.annotations.find((p) => p.track_id === a.track_id && p.bbox_3d); - if (prevMatch?.bbox_3d) pts.push([prevMatch.bbox_3d[0], prevMatch.bbox_3d[1]]); - pts.push([a.bbox_3d[0], a.bbox_3d[1]]); - const nextMatch = nextFrame?.annotations.find((n) => n.track_id === a.track_id && n.bbox_3d); - if (nextMatch?.bbox_3d) pts.push([nextMatch.bbox_3d[0], nextMatch.bbox_3d[1]]); - if (pts.length > 1) { - trails.push({ trackId: a.track_id, pts, highlighted: a.annotation_id === highlightId }); - } - } - - return ( - - {trails.map((tr) => ( - t.toSvg(p).map((v) => v.toFixed(1)).join(',')).join(' ')} - fill="none" - stroke={tr.highlighted ? '#ef5350' : '#7e57c2'} - strokeWidth={tr.highlighted ? 2 : 1.2} - strokeOpacity={0.85} - /> - ))} - {frame.annotations.map((a) => { - if (!a.bbox_3d) return null; - const [x, y] = t.toSvg([a.bbox_3d[0], a.bbox_3d[1]]); - const highlighted = a.annotation_id === highlightId; - return ( - - - - {a.class_name} - - - ); - })} - {/* ego marker */} - - - - - EGO - - - ); -} - -function MiniBev({ frame, highlightTrack, label }: { frame: FrameSummary | null; highlightTrack: string | null; label: string }) { - const W = 140; - const H = 220; - if (!frame) { - return ( - - - - n/a - - - - {label} - - - ); - } - const t = makeBevTransform(frame, W, H, 8); - return ( - - - {frame.lidar_points_bev.map((p, i) => { - const [x, y] = t.toSvg([p[0], p[1]]); - return ; - })} - {frame.annotations.map((a) => { - if (!a.bbox_3d) return null; - const highlighted = highlightTrack !== null && a.track_id === highlightTrack; - return ( - - ); - })} - - - {label} - - - ); -} - -// ---------------------------------------------------------------- main component - -export default function HITLReview({ - tasks, - selectedTask, - onSelectTask, - frame, - prevFrame, - nextFrame, - frameLoading, -}: { - tasks: ReviewTask[]; - selectedTask: ReviewTask | null; - onSelectTask: (taskId: string) => void; - frame: FrameSummary | null; - prevFrame: FrameSummary | null; - nextFrame: FrameSummary | null; - frameLoading: boolean; -}) { - const evidence = selectedTask?.evidence ?? null; - const annotation: Annotation | null = - (frame && selectedTask ? frame.annotations.find((a) => a.annotation_id === selectedTask.annotation_id) : null) ?? - null; - const highlightId = selectedTask?.annotation_id ?? ''; - - return ( - - {/* Left: task list */} - - {tasks.length === 0 ? ( - - No review tasks — flagged labels will appear here. - - ) : ( - - {tasks.map((t) => ( - onSelectTask(t.task_id)} - sx={{ display: 'block', borderBottom: '1px solid #1c232b', px: 1 }} - > - - - {t.annotation_id} - - - - - {t.failure_reasons.map((r) => ( - - ))} - - - ))} - - )} - - - {/* Right: sensor views + evidence */} - - {!selectedTask ? ( - - - Select a task to inspect its frame across camera, LiDAR, BEV and temporal views. - - - ) : frameLoading || !frame ? ( - - {frameLoading ? ( - - ) : ( - - Frame data unavailable (backend offline?). - - )} - - ) : ( - - - - - - - - - - - - - - - - - - - - - )} - - {selectedTask ? ( - - {evidence ? ( - <> - - - - - - - - - - - ) : ( - - No evaluation evidence attached to this task. - - )} - - ) : null} - - - ); -} diff --git a/hf-space/src/components/labeleval/HaystackVisualizer.tsx b/hf-space/src/components/labeleval/HaystackVisualizer.tsx deleted file mode 100644 index 7148d2539b33f0e5967e24e65f38b76da8eb44f3..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/HaystackVisualizer.tsx +++ /dev/null @@ -1,191 +0,0 @@ -import { useMemo, useState } from 'react'; -import Box from '@mui/material/Box'; -import Typography from '@mui/material/Typography'; -import type { HaystackCategory, HaystackPoint } from '../../types/labeleval'; -import { SectionCard, fmtInt } from './shared'; - -const WIDTH = 700; -const HEIGHT = 420; -const PAD = 24; - -const CATEGORY_STYLE: Record = { - normal: { color: '#5c6773', label: 'Normal' }, - anomaly: { color: '#ffa726', label: 'Anomaly' }, - rare_event: { color: '#ef5350', label: 'Rare event' }, - false_positive: { color: '#ab47bc', label: 'False positive' }, - false_negative: { color: '#42a5f5', label: 'False negative' }, - verified: { color: '#66bb6a', label: 'Verified' }, -}; - -const CATEGORY_ORDER: HaystackCategory[] = [ - 'normal', - 'verified', - 'false_positive', - 'false_negative', - 'anomaly', - 'rare_event', -]; - -export default function HaystackVisualizer({ - points, - onPointClick, -}: { - points: HaystackPoint[]; - onPointClick: (point: HaystackPoint) => void; -}) { - const [hovered, setHovered] = useState(null); - - const { scaled, counts } = useMemo(() => { - const counts = new Map(); - for (const p of points) counts.set(p.category, (counts.get(p.category) ?? 0) + 1); - if (points.length === 0) return { scaled: [] as Array, counts }; - let minX = Infinity; - let maxX = -Infinity; - let minY = Infinity; - let maxY = -Infinity; - for (const p of points) { - if (p.x < minX) minX = p.x; - if (p.x > maxX) maxX = p.x; - if (p.y < minY) minY = p.y; - if (p.y > maxY) maxY = p.y; - } - const spanX = maxX - minX || 1; - const spanY = maxY - minY || 1; - const scaled = points.map((p) => ({ - ...p, - sx: PAD + ((p.x - minX) / spanX) * (WIDTH - 2 * PAD), - sy: HEIGHT - PAD - ((p.y - minY) / spanY) * (HEIGHT - 2 * PAD), - })); - // Draw interesting categories on top of the normal haystack. - scaled.sort((a, b) => CATEGORY_ORDER.indexOf(a.category) - CATEGORY_ORDER.indexOf(b.category)); - return { scaled, counts }; - }, [points]); - - return ( - - - - - {/* subtle grid */} - {Array.from({ length: 7 }, (_, i) => ( - - ))} - {Array.from({ length: 5 }, (_, i) => ( - - ))} - {scaled.map((p) => { - const style = CATEGORY_STYLE[p.category]; - const interesting = p.category !== 'normal'; - const common = { - cursor: 'pointer', - onClick: () => onPointClick(p), - onMouseEnter: () => setHovered(p), - onMouseLeave: () => setHovered(null), - }; - if (p.category === 'rare_event') { - const r = 6; - return ( - - ); - } - return ( - - ); - })} - {points.length === 0 ? ( - - No haystack points yet — run the evaluation pipeline. - - ) : null} - - {hovered ? ( - - - {hovered.id} - - - {hovered.class_name} · {CATEGORY_STYLE[hovered.category].label} · score{' '} - {hovered.anomaly_score.toFixed(3)} · {hovered.frame_id} - - - ) : null} - - - - - LEGEND - - {CATEGORY_ORDER.map((cat) => { - const style = CATEGORY_STYLE[cat]; - return ( - - {cat === 'rare_event' ? ( - - ) : ( - - )} - - {style.label} - - - {fmtInt(counts.get(cat) ?? 0)} - - - ); - })} - - Click any point to open its full evidence panel. - - - - - ); -} diff --git a/hf-space/src/components/labeleval/InputStage.tsx b/hf-space/src/components/labeleval/InputStage.tsx deleted file mode 100644 index 44d4ddb65bb54f0530586f2758e6ecd733e1278a..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/InputStage.tsx +++ /dev/null @@ -1,51 +0,0 @@ -import Box from '@mui/material/Box'; -import Typography from '@mui/material/Typography'; -import Card from '@mui/material/Card'; -import CardContent from '@mui/material/CardContent'; -import { Radar, Camera, Map as MapIcon, Compass, Activity, Clock, Route } from 'lucide-react'; -import type { DatasetSummary } from '../../types/labeleval'; -import { SectionCard, fmtInt } from './shared'; - -export default function InputStage({ dataset }: { dataset: DatasetSummary | null }) { - const frames = dataset?.num_frames ?? 0; - const annotations = dataset?.num_annotations ?? 0; - const sequences = dataset?.num_sequences ?? 0; - const scenes = dataset?.num_scenes ?? 0; - - const modalities = [ - { name: 'LiDAR', icon: , detail: `${fmtInt(frames)} sweeps (1 per frame)` }, - { name: 'Camera', icon: , detail: `${fmtInt(frames)} images (1 per frame)` }, - { name: 'Maps', icon: , detail: `${fmtInt(scenes)} HD map tiles (1 per scene)` }, - { name: 'Ego Pose', icon: , detail: `${fmtInt(frames)} poses @ frame rate` }, - { name: 'Telemetry', icon: , detail: `${fmtInt(frames)} CAN/IMU samples` }, - { name: 'Timestamps', icon: , detail: `${fmtInt(frames)} synchronized (µs)` }, - { name: 'Tracks', icon: , detail: `${fmtInt(annotations)} annotations with track ids` }, - ]; - - return ( - - - Hierarchy: Dataset ({dataset ? dataset.name : 'none selected'}) → Scene ( - {fmtInt(scenes)}) → Sequence ({fmtInt(sequences)}) → Frame ({fmtInt(frames)} - ). Every frame bundles all sensor modalities below, time-synchronized, before auto-labeling. - - - {modalities.map((m) => ( - - - - {m.icon} - - {m.name} - - - - {m.detail} - - - - ))} - - - ); -} diff --git a/hf-space/src/components/labeleval/LabelEvaluationEngine.tsx b/hf-space/src/components/labeleval/LabelEvaluationEngine.tsx deleted file mode 100644 index bad15540a5299cd4e237eabdd7cbe91cf2edd356..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/LabelEvaluationEngine.tsx +++ /dev/null @@ -1,88 +0,0 @@ -import Box from '@mui/material/Box'; -import Card from '@mui/material/Card'; -import CardContent from '@mui/material/CardContent'; -import Typography from '@mui/material/Typography'; -import { BrainCircuit, TrendingDown, Users } from 'lucide-react'; -import type { PipelineStateResponse, ServiceStatus } from '../../types/labeleval'; -import { SectionCard, StatusChip, fmtInt } from './shared'; - -function findService(pipeline: PipelineStateResponse | null, ...needles: string[]): ServiceStatus | null { - if (!pipeline) return null; - const lower = needles.map((n) => n.toLowerCase()); - return ( - pipeline.services.find((s) => { - const name = s.service.toLowerCase(); - return lower.some((n) => name.includes(n)); - }) ?? null - ); -} - -function EngineCard({ - icon, - title, - service, - description, -}: { - icon: React.ReactNode; - title: string; - service: ServiceStatus | null; - description: string; -}) { - return ( - - - - {icon} - - {title} - - - - - {description} - - {service ? ( - - {fmtInt(service.processed)} / {fmtInt(service.total)} processed · {fmtInt(service.process_units)} process - units{service.detail ? ` · ${service.detail}` : ''} - - ) : ( - - No live status yet. - - )} - - - ); -} - -export default function LabelEvaluationEngine({ pipeline }: { pipeline: PipelineStateResponse | null }) { - return ( - - - Each engine produces independent evidence that feeds the Quality Gate. There is never a single - blended score: a label is only verified when every applicable gate passes on its own evidence. - - - } - title="ML Anomaly Detection" - service={findService(pipeline, 'anomaly')} - description="Ensemble of classical + deep detectors scoring every annotation for statistical abnormality relative to the fleet distribution." - /> - } - title="Regression Tracking" - service={findService(pipeline, 'regression')} - description="Compares the current model's per-class and per-scenario metrics against the baseline; any tolerated-delta breach is evidence." - /> - } - title="Grader Disagreement" - service={findService(pipeline, 'grader', 'consensus', 'disagreement')} - description="Multiple automated graders vote on class, spatial and temporal correctness; low consensus is evidence of an unreliable label." - /> - - - ); -} diff --git a/hf-space/src/components/labeleval/QueueStatus.tsx b/hf-space/src/components/labeleval/QueueStatus.tsx deleted file mode 100644 index 7a9354f479686388a9a3c8279d81296f4476d279..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/QueueStatus.tsx +++ /dev/null @@ -1,57 +0,0 @@ -import Box from '@mui/material/Box'; -import Typography from '@mui/material/Typography'; -import Chip from '@mui/material/Chip'; -import type { QueueStatus as QueueStatusType } from '../../types/labeleval'; -import { getQueueStatus, usePoll } from '../../services/labeleval'; -import { SectionCard, MetricCard, HBar, fmtInt, fmtNum, ErrorNote } from './shared'; - -export default function QueueStatus({ queue: queueProp }: { queue?: QueueStatusType | null }) { - // Self-fetches unless a queue snapshot is supplied (e.g. from the SSE stream). - const { data, error } = usePoll(getQueueStatus, queueProp === undefined ? 3000 : null, [ - queueProp === undefined, - ]); - const queue = queueProp !== undefined ? queueProp : data; - - const topics = queue ? Object.entries(queue.depth_by_topic) : []; - const maxDepth = Math.max(1, ...topics.map(([, depth]) => depth)); - - return ( - : null} - > - {!queue && error ? : null} - {queue ? ( - <> - - - - - - - - {topics.length > 0 ? ( - <> - - DEPTH BY TOPIC - - - {topics.map(([topic, depth]) => ( - - ))} - - - ) : ( - - No topic depth data. - - )} - - ) : !error ? ( - - Waiting for queue data… - - ) : null} - - ); -} diff --git a/hf-space/src/components/labeleval/RareEventConfig.tsx b/hf-space/src/components/labeleval/RareEventConfig.tsx deleted file mode 100644 index e741a982529d1a7bed83c0c30b77de734de40ccf..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/RareEventConfig.tsx +++ /dev/null @@ -1,386 +0,0 @@ -import { useEffect, useState } from 'react'; -import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import Slider from '@mui/material/Slider'; -import Select from '@mui/material/Select'; -import MenuItem from '@mui/material/MenuItem'; -import Switch from '@mui/material/Switch'; -import TextField from '@mui/material/TextField'; -import Typography from '@mui/material/Typography'; -import Alert from '@mui/material/Alert'; -import CircularProgress from '@mui/material/CircularProgress'; -import { Save, Play } from 'lucide-react'; -import type { AnomalyConfig, EnsembleStrategy } from '../../types/labeleval'; -import { getAnomalyConfig, postAnomalyConfig, runPipeline } from '../../services/labeleval'; -import { SectionCard } from './shared'; -import AnomalyDetectionConfig from './AnomalyDetectionConfig'; - -const DEFAULT_CONFIG: AnomalyConfig = { - imbalance: { method: 'smote', minority_boost: 2 }, - detectors: { - knn: { enabled: true, k: 5 }, - lof: { enabled: true, n_neighbors: 20 }, - isolation_forest: { enabled: true, n_estimators: 100 }, - ocsvm: { enabled: false, nu: 0.05 }, - dbscan: { enabled: false, eps: 0.5, min_samples: 5 }, - }, - deep: { - autoencoder: { enabled: true, latent_dim: 16, epochs: 20 }, - vae: { enabled: false, latent_dim: 8 }, - gan: { enabled: false }, - reconstruction_threshold: 0.5, - }, - advanced: { - few_shot: { enabled: false, support_per_class: 5 }, - ensemble_strategy: 'weighted_average', - decision_threshold: 0.6, - }, -}; - -function Column({ title, children }: { title: string; children: React.ReactNode }) { - return ( - - - {title} - - {children} - - ); -} - -export default function RareEventConfig({ - activeDatasetId, - onRunStarted, -}: { - activeDatasetId: string | null; - onRunStarted?: (runId: string) => void; -}) { - const [config, setConfig] = useState(DEFAULT_CONFIG); - const [loading, setLoading] = useState(true); - const [busy, setBusy] = useState<'save' | 'run' | null>(null); - const [notice, setNotice] = useState<{ severity: 'success' | 'error' | 'warning'; text: string } | null>(null); - - useEffect(() => { - let cancelled = false; - getAnomalyConfig() - .then((c) => { - if (!cancelled) setConfig(c); - }) - .catch(() => { - /* backend offline — keep defaults, save will still POST */ - }) - .finally(() => { - if (!cancelled) setLoading(false); - }); - return () => { - cancelled = true; - }; - }, []); - - const save = async () => { - setBusy('save'); - setNotice(null); - try { - const saved = await postAnomalyConfig(config); - setConfig(saved); - setNotice({ severity: 'success', text: 'Configuration saved.' }); - } catch (err) { - setNotice({ severity: 'error', text: err instanceof Error ? err.message : String(err) }); - } finally { - setBusy(null); - } - }; - - const rerun = async () => { - if (!activeDatasetId) { - setNotice({ severity: 'warning', text: 'Select or generate a dataset first (Datasets page).' }); - return; - } - setBusy('run'); - setNotice(null); - try { - const res = await runPipeline(activeDatasetId); - setNotice({ severity: 'success', text: `Evaluation run ${res.run_id} started (${res.status}).` }); - onRunStarted?.(res.run_id); - } catch (err) { - setNotice({ severity: 'error', text: err instanceof Error ? err.message : String(err) }); - } finally { - setBusy(null); - } - }; - - return ( - - - - - } - > - {notice ? ( - setNotice(null)} sx={{ mb: 1 }}> - {notice.text} - - ) : null} - - {/* Column 1: imbalanced data */} - - - Method - - - - Minority boost ×{config.imbalance.minority_boost.toFixed(1)} - - - setConfig({ ...config, imbalance: { ...config.imbalance, minority_boost: v as number } }) - } - /> - - - {/* Column 2: classical anomaly detectors */} - - setConfig({ ...config, detectors })} - /> - - - {/* Column 3: deep learning */} - - - - Autoencoder - - - setConfig({ - ...config, - deep: { ...config.deep, autoencoder: { ...config.deep.autoencoder, enabled: e.target.checked } }, - }) - } - /> - - {config.deep.autoencoder.enabled ? ( - - - setConfig({ - ...config, - deep: { - ...config.deep, - autoencoder: { ...config.deep.autoencoder, latent_dim: Number(e.target.value) }, - }, - }) - } - sx={{ flex: 1 }} - /> - - setConfig({ - ...config, - deep: { - ...config.deep, - autoencoder: { ...config.deep.autoencoder, epochs: Number(e.target.value) }, - }, - }) - } - sx={{ flex: 1 }} - /> - - ) : null} - - - VAE - - - setConfig({ - ...config, - deep: { ...config.deep, vae: { ...config.deep.vae, enabled: e.target.checked } }, - }) - } - /> - - {config.deep.vae.enabled ? ( - - setConfig({ - ...config, - deep: { ...config.deep, vae: { ...config.deep.vae, latent_dim: Number(e.target.value) } }, - }) - } - sx={{ mb: 1, width: '100%' }} - /> - ) : null} - - - GAN - - - setConfig({ ...config, deep: { ...config.deep, gan: { enabled: e.target.checked } } }) - } - /> - - - Reconstruction threshold {config.deep.reconstruction_threshold.toFixed(2)} - - - setConfig({ ...config, deep: { ...config.deep, reconstruction_threshold: v as number } }) - } - /> - - - {/* Column 4: advanced / ensemble */} - - - - Few-shot - - - setConfig({ - ...config, - advanced: { - ...config.advanced, - few_shot: { ...config.advanced.few_shot, enabled: e.target.checked }, - }, - }) - } - /> - - {config.advanced.few_shot.enabled ? ( - - setConfig({ - ...config, - advanced: { - ...config.advanced, - few_shot: { ...config.advanced.few_shot, support_per_class: Number(e.target.value) }, - }, - }) - } - sx={{ mb: 1, width: '100%' }} - /> - ) : null} - - Ensemble strategy - - - - Decision threshold {config.advanced.decision_threshold.toFixed(2)} - - - setConfig({ ...config, advanced: { ...config.advanced, decision_threshold: v as number } }) - } - /> - - - - ); -} diff --git a/hf-space/src/components/labeleval/RegressionTracking.tsx b/hf-space/src/components/labeleval/RegressionTracking.tsx deleted file mode 100644 index 3460aadaca57f53600289a7c1bb2fa8779ab84cc..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/RegressionTracking.tsx +++ /dev/null @@ -1,186 +0,0 @@ -import { Fragment, useState } from 'react'; -import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import Chip from '@mui/material/Chip'; -import Collapse from '@mui/material/Collapse'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import { AlertTriangle, ChevronDown, ChevronRight, Sparkles } from 'lucide-react'; -import type { CopilotExplainRequest, RegressionEntry, RegressionResponse } from '../../types/labeleval'; -import { SectionCard, fmtNum } from './shared'; - -function DeltaTable({ entry }: { entry: RegressionEntry }) { - return ( - - - - Metric - Baseline - Current - Delta - Tolerance - - - - {entry.deltas.map((d) => ( - - {d.metric} - - {fmtNum(d.baseline)} - - - {fmtNum(d.current)} - - - {d.delta > 0 ? '+' : ''} - {fmtNum(d.delta)} - - - ±{fmtNum(d.tolerance)} - - - ))} - -
    - ); -} - -export default function RegressionTracking({ - regression, - onAskCopilot, -}: { - regression: RegressionResponse | null; - onAskCopilot: (request: CopilotExplainRequest) => void; -}) { - const [expanded, setExpanded] = useState(null); - - return ( - <> - {regression?.current_alert ? ( - - - - - REGRESSION DETECTED - - - The current model performs worse than its baseline beyond configured tolerances. Affected labels are - blocked from auto-verification until resolved. - - - - ) : null} - - - {!regression || regression.entries.length === 0 ? ( - - No regression comparisons yet — they appear after evaluating a model that has a baseline. - - ) : ( - - - - - Model - Baseline - Dataset - Date - Kinds - Affected classes - Affected scenarios - Status - - - - - {regression.entries.map((e) => { - const key = e.run_id; - const isOpen = expanded === key; - return ( - - setExpanded(isOpen ? null : key)} sx={{ cursor: 'pointer' }}> - - {isOpen ? : } - - {e.model_version} - - {e.baseline_version ?? '—'} - - {e.dataset_version} - {new Date(e.date).toLocaleString()} - - - {e.kinds.map((k) => ( - - ))} - - - {e.affected_classes.join(', ') || '—'} - {e.affected_scenarios.join(', ') || '—'} - - - - ev.stopPropagation()}> - {e.regression_detected ? ( - - ) : null} - - - - - - - - - - - - - ); - })} - -
    - )} -     - - ); -} diff --git a/hf-space/src/components/labeleval/RelabelingPanel.tsx b/hf-space/src/components/labeleval/RelabelingPanel.tsx deleted file mode 100644 index 81a875c3e2d49b6496937fd54ee5dfd85e4f6358..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/RelabelingPanel.tsx +++ /dev/null @@ -1,243 +0,0 @@ -import { useEffect, useState } from 'react'; -import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import Dialog from '@mui/material/Dialog'; -import DialogTitle from '@mui/material/DialogTitle'; -import DialogContent from '@mui/material/DialogContent'; -import DialogActions from '@mui/material/DialogActions'; -import TextField from '@mui/material/TextField'; -import Select from '@mui/material/Select'; -import MenuItem from '@mui/material/MenuItem'; -import InputLabel from '@mui/material/InputLabel'; -import FormControl from '@mui/material/FormControl'; -import Typography from '@mui/material/Typography'; -import Alert from '@mui/material/Alert'; -import CircularProgress from '@mui/material/CircularProgress'; -import { Check, X, Wrench, Merge, Split, EyeOff } from 'lucide-react'; -import type { Annotation, ReviewAction, ReviewActionResponse, ReviewTask } from '../../types/labeleval'; -import { postReviewAction } from '../../services/labeleval'; -import { SectionCard, GateLineList, StatusChip } from './shared'; - -const CLASSES = ['pedestrian', 'cyclist', 'vehicle', 'motorcycle', 'truck']; -const BBOX_FIELDS = ['x', 'y', 'z', 'l', 'w', 'h', 'yaw'] as const; - -export default function RelabelingPanel({ - task, - annotation, - onResolved, -}: { - task: ReviewTask | null; - annotation: Annotation | null; - onResolved: () => void; -}) { - const [busy, setBusy] = useState(null); - const [errorMsg, setErrorMsg] = useState(null); - const [result, setResult] = useState(null); - const [correctOpen, setCorrectOpen] = useState(false); - const [bbox, setBbox] = useState([0, 0, 0, 4, 2, 1.6, 0]); - const [klass, setKlass] = useState('vehicle'); - const [mergeOpen, setMergeOpen] = useState(false); - const [mergeTrackId, setMergeTrackId] = useState(''); - - // Clear the previous re-validation result when switching tasks. - useEffect(() => { - setResult(null); - setErrorMsg(null); - }, [task?.task_id]); - - const submit = async (action: ReviewAction, extra?: { corrected_bbox_3d?: number[]; corrected_class?: string; merge_with_track_id?: string }) => { - if (!task) return; - setBusy(action); - setErrorMsg(null); - try { - const res = await postReviewAction(task.task_id, { action, ...extra }); - setResult(res); - setCorrectOpen(false); - setMergeOpen(false); - onResolved(); - } catch (err) { - setErrorMsg(err instanceof Error ? err.message : String(err)); - } finally { - setBusy(null); - } - }; - - const openCorrect = () => { - setBbox(annotation?.bbox_3d ? [...annotation.bbox_3d] : [0, 0, 0, 4, 2, 1.6, 0]); - setKlass(annotation?.class_name && CLASSES.includes(annotation.class_name) ? annotation.class_name : 'vehicle'); - setCorrectOpen(true); - }; - - const revalidationPassed = result?.revalidation.validation.passed ?? null; - - return ( - - - Corrected labels are never verified without re-running the quality gates — every action below triggers - re-validation. - - - {errorMsg ? ( - setErrorMsg(null)} sx={{ mb: 1 }}> - {errorMsg} - - ) : null} - - - - - - - - - - - {result ? ( - - - - {result.message} - - {result.task.resolution?.final_status ? : null} - - - Re-validation re-ran the quality gates:{' '} - - {revalidationPassed ? 'PASSED → VERIFIED' : 'FAILED'} - - {!revalidationPassed && result.revalidation.decision?.primary_failure_reason - ? ` — ${result.revalidation.decision.primary_failure_reason.replace(/_/g, ' ')}` - : ''} - - - - ) : null} - - {/* Correct dialog */} - setCorrectOpen(false)} maxWidth="xs" fullWidth> - Correct annotation {task?.annotation_id} - - - Class - - - - {BBOX_FIELDS.map((field, i) => ( - { - const next = [...bbox]; - next[i] = Number(e.target.value); - setBbox(next); - }} - /> - ))} - - - - - - - - - {/* Merge dialog */} - setMergeOpen(false)} maxWidth="xs" fullWidth> - Merge tracks - - setMergeTrackId(e.target.value)} - helperText={annotation?.track_id ? `Current track: ${annotation.track_id}` : undefined} - /> - - - - - - - - ); -} diff --git a/hf-space/src/components/labeleval/StrictQualityValidation.tsx b/hf-space/src/components/labeleval/StrictQualityValidation.tsx deleted file mode 100644 index 473bdc267eb69452097d476205a4e97fea97dd23..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/StrictQualityValidation.tsx +++ /dev/null @@ -1,93 +0,0 @@ -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import type { QualityMetrics } from '../../types/labeleval'; -import { SectionCard, fmtNum } from './shared'; - -interface CheckRow { - check: string; - tolerance: string; - observed: (m: QualityMetrics) => string; -} - -/** - * The geometric validation checks with their configured tolerances. - * The API only exposes aggregate metrics, so the "observed" column surfaces - * the closest aggregate where one exists. - */ -const CHECKS: CheckRow[] = [ - { - check: 'Point-in-box ratio', - tolerance: '≥ 0.60 of box volume occupied by LiDAR returns', - observed: () => '— (per-annotation, see evidence panels)', - }, - { - check: 'Point density', - tolerance: '≥ 5 pts/m³ at range-adjusted threshold', - observed: () => '— (per-annotation, see evidence panels)', - }, - { - check: 'Occupancy', - tolerance: 'No overlap > 0.30 IoU with another box of same class', - observed: (m) => `mean 3D IoU ${fmtNum(m.global.mean_iou_3d)}`, - }, - { - check: 'Dimensions', - tolerance: 'Within class prior ± 2.5σ (l, w, h)', - observed: () => '— (per-annotation, see evidence panels)', - }, - { - check: 'Orientation', - tolerance: '≤ 15° error vs. reference / motion heading', - observed: (m) => `mean ${fmtNum(m.global.mean_orientation_error_deg, 1)}°`, - }, - { - check: 'Centroid', - tolerance: '≤ 0.5 m position error vs. reference', - observed: (m) => `mean ${fmtNum(m.global.mean_position_error)} m`, - }, - { - check: 'Ground contact', - tolerance: 'Box bottom within ± 0.3 m of ground plane', - observed: () => '— (per-annotation, see evidence panels)', - }, - { - check: 'Sensor consistency', - tolerance: 'LiDAR vs. camera class/extent agreement required', - observed: (m) => `anomaly rate ${(m.global.anomaly_rate * 100).toFixed(1)}%`, - }, -]; - -export default function StrictQualityValidation({ metrics }: { metrics: QualityMetrics | null }) { - return ( - - - Every annotation must pass all applicable geometric checks. Failing any single gate flags the label — gates are - never averaged away. - - - - - Check - Configured tolerance - Observed aggregate - - - - {CHECKS.map((c) => ( - - {c.check} - {c.tolerance} - - {metrics ? c.observed(metrics) : '—'} - - - ))} - -
    -     - ); -} diff --git a/hf-space/src/components/labeleval/TrainingFlywheel.tsx b/hf-space/src/components/labeleval/TrainingFlywheel.tsx deleted file mode 100644 index f85c8cb612b9c255a5e8689f8491707ce41a8a0a..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/TrainingFlywheel.tsx +++ /dev/null @@ -1,99 +0,0 @@ -import Typography from '@mui/material/Typography'; -import { SectionCard, fmtInt } from './shared'; - -interface FlywheelCounts { - verified: number; - datasets: number; - activeJobs: number; - models: number; - evaluated: number; -} - -const NODES = [ - { key: 'verified', label: 'Verified Data' }, - { key: 'datasets', label: 'Training Dataset' }, - { key: 'activeJobs', label: 'Train' }, - { key: 'models', label: 'New Model' }, - { key: 'evaluated', label: 'New Evaluations' }, -] as const; - -export default function TrainingFlywheel({ counts }: { counts: FlywheelCounts }) { - const W = 520; - const H = 380; - const cx = W / 2; - const cy = H / 2; - const R = 135; - const nodeR = 44; - - const positions = NODES.map((_, i) => { - const angle = -Math.PI / 2 + (i * 2 * Math.PI) / NODES.length; - return { x: cx + R * Math.cos(angle), y: cy + R * Math.sin(angle) }; - }); - - return ( - - - - - - - - {/* edges around the circle */} - {positions.map((p, i) => { - const next = positions[(i + 1) % positions.length]!; - const dx = next.x - p.x; - const dy = next.y - p.y; - const dist = Math.hypot(dx, dy) || 1; - const startX = p.x + (dx / dist) * (nodeR + 4); - const startY = p.y + (dy / dist) * (nodeR + 4); - const endX = next.x - (dx / dist) * (nodeR + 8); - const endY = next.y - (dy / dist) * (nodeR + 8); - // bow the edge outward for a circular feel - const mx = (startX + endX) / 2 + ((startX + endX) / 2 - cx) * 0.22; - const my = (startY + endY) / 2 + ((startY + endY) / 2 - cy) * 0.22; - return ( - - ); - })} - {/* nodes */} - {NODES.map((node, i) => { - const p = positions[i]!; - const value = counts[node.key]; - return ( - - - - {node.label.split(' ').map((word, j) => ( - - {word} - - ))} - - - {fmtInt(value)} - - - ); - })} - - verified labels feed - - - better models feed better labels - - - - Only VERIFIED labels enter training datasets; every new model is evaluated (and regression-checked) before its - outputs feed back into the loop. - - - ); -} diff --git a/hf-space/src/components/labeleval/TrainingLogViewer.tsx b/hf-space/src/components/labeleval/TrainingLogViewer.tsx deleted file mode 100644 index 2cd35897980937601092fb63db8575d38b170f0a..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/TrainingLogViewer.tsx +++ /dev/null @@ -1,42 +0,0 @@ -import { useEffect, useRef } from 'react'; -import Box from '@mui/material/Box'; -import Typography from '@mui/material/Typography'; - -export default function TrainingLogViewer({ logs }: { logs: string[] }) { - const boxRef = useRef(null); - - useEffect(() => { - const el = boxRef.current; - if (el) el.scrollTop = el.scrollHeight; - }, [logs.length]); - - return ( - - - TRAINING LOG - - - {logs.length === 0 ? ( - No log output yet… - ) : ( - logs.map((line, i) =>
    {line}
    ) - )} -     -     - ); -} diff --git a/hf-space/src/components/labeleval/TriageDecisionPanel.tsx b/hf-space/src/components/labeleval/TriageDecisionPanel.tsx deleted file mode 100644 index a6bec42a6e8653d1e1c2b78fece16cb88071211f..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/TriageDecisionPanel.tsx +++ /dev/null @@ -1,139 +0,0 @@ -import { useState } from 'react'; -import Box from '@mui/material/Box'; -import Chip from '@mui/material/Chip'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import type { TriageDecision } from '../../types/labeleval'; -import { SectionCard, StatusChip, GateLineList, HBar, fmtNum } from './shared'; - -export default function TriageDecisionPanel({ decisions }: { decisions: TriageDecision[] }) { - const [selectedId, setSelectedId] = useState(null); - const selected = decisions.find((d) => d.decision_id === selectedId) ?? null; - - const policy = decisions[0] ?? null; - - const reasonCounts = new Map(); - for (const d of decisions) { - for (const r of d.failure_reasons) reasonCounts.set(r, (reasonCounts.get(r) ?? 0) + 1); - } - const reasonRows = [...reasonCounts.entries()].sort((a, b) => b[1] - a[1]); - const maxReason = Math.max(1, ...reasonRows.map(([, c]) => c)); - - return ( - - - {policy ? ( - <> - - - {Object.entries(policy.policy_values).map(([k, v]) => ( - - ))} - - - ) : ( - - No triage decisions yet — the policy appears after the first evaluation run. - - )} - - - - - {decisions.length === 0 ? ( - - No decisions to show. - - ) : ( - - - - Annotation - Status - Primary failure reason - Decided - - - - {decisions.map((d) => ( - setSelectedId(d.decision_id)} - sx={{ cursor: 'pointer' }} - > - {d.annotation_id} - - - - - {d.primary_failure_reason ? d.primary_failure_reason.replace(/_/g, ' ') : '—'} - - {new Date(d.decided_at).toLocaleTimeString()} - - ))} - -
    - )} -     - - - - {selected ? ( - <> - - - {selected.annotation_id} - - - - {selected.primary_failure_reason ? ( - - Primary failure: {selected.primary_failure_reason.replace(/_/g, ' ')} - - ) : ( - - All applicable gates passed. - - )} - - - ) : ( - - Select a decision to see its per-gate ✓/✗ breakdown (actual vs. threshold). - - )} - - - - {reasonRows.length === 0 ? ( - - No failures recorded. - - ) : ( - reasonRows.map(([reason, count]) => ( - - )) - )} - - -     -     - ); -} diff --git a/hf-space/src/components/labeleval/VerificationFunnel.tsx b/hf-space/src/components/labeleval/VerificationFunnel.tsx deleted file mode 100644 index a44ad73dee06e7c57177a969c38fa3dfcaee542f..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/VerificationFunnel.tsx +++ /dev/null @@ -1,71 +0,0 @@ -import Box from '@mui/material/Box'; -import Typography from '@mui/material/Typography'; -import type { FunnelResponse, FunnelStage } from '../../types/labeleval'; -import { SectionCard, fmtInt, fmtPct } from './shared'; - -function StageBar({ stage, max, color }: { stage: FunnelStage; max: number; color: string }) { - const width = max > 0 ? Math.max(2, (stage.count / max) * 100) : 2; - return ( - - - - {stage.stage.replace(/_/g, ' ')} - - - {fmtInt(stage.count)} · {fmtPct(stage.pct_of_input)} - - - - - - - ); -} - -export default function VerificationFunnel({ funnel }: { funnel: FunnelResponse | null }) { - if (!funnel || (funnel.main_path.length === 0 && funnel.side_path.length === 0)) { - return ( - - - No funnel data yet — run the evaluation pipeline to populate it. - - - ); - } - const maxMain = Math.max(1, ...funnel.main_path.map((s) => s.count)); - const maxSide = Math.max(1, ...funnel.side_path.map((s) => s.count)); - return ( - - - - - MAIN PATH - - - {funnel.main_path.map((s) => ( - - ))} - - - - - SIDE BRANCH — FLAGGED → HITL → RE-LABEL → RE-VALIDATION → VERIFIED - - - {funnel.side_path.map((s) => ( - - ))} - - - - - ); -} diff --git a/hf-space/src/components/labeleval/shared.tsx b/hf-space/src/components/labeleval/shared.tsx deleted file mode 100644 index b2a2480aebb36e57d9d7dbc4af2ce9c114a11f9b..0000000000000000000000000000000000000000 --- a/hf-space/src/components/labeleval/shared.tsx +++ /dev/null @@ -1,278 +0,0 @@ -import type { ReactNode } from 'react'; -import Box from '@mui/material/Box'; -import Card from '@mui/material/Card'; -import CardContent from '@mui/material/CardContent'; -import Chip from '@mui/material/Chip'; -import Typography from '@mui/material/Typography'; -import Alert from '@mui/material/Alert'; -import CircularProgress from '@mui/material/CircularProgress'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import { CheckCircle2, XCircle, MinusCircle } from 'lucide-react'; -import type { GateLine } from '../../types/labeleval'; - -// ---------------------------------------------------------------- formatting - -export function fmtInt(n: number | null | undefined): string { - if (n === null || n === undefined || Number.isNaN(n)) return '—'; - return Math.round(n).toLocaleString(); -} - -export function fmtNum(n: number | null | undefined, digits = 3): string { - if (n === null || n === undefined || Number.isNaN(n)) return '—'; - return n.toLocaleString(undefined, { maximumFractionDigits: digits }); -} - -/** - * Formats a rate as a percentage with 1 decimal. Values <= 1 are treated as - * fractions (0..1); values > 1 are assumed to already be percentages. - */ -export function fmtPct(v: number | null | undefined): string { - if (v === null || v === undefined || Number.isNaN(v)) return '—'; - const pct = v <= 1 ? v * 100 : v; - return `${pct.toFixed(1)}%`; -} - -export function pctFraction(v: number | null | undefined): number { - if (v === null || v === undefined || Number.isNaN(v)) return 0; - return v <= 1 ? v : v / 100; -} - -// ---------------------------------------------------------------- MetricCard - -export function MetricCard({ - label, - value, - sub, - accent, -}: { - label: string; - value: ReactNode; - sub?: ReactNode; - accent?: string; -}) { - return ( - - - - {label} - - - {value} - - {sub ? ( - - {sub} - - ) : null} - - - ); -} - -// ---------------------------------------------------------------- StatusChip - -const STATUS_COLORS: Record = { - // triage statuses - AUTO_GRADED: { bg: '#1b5e20', fg: '#a5d6a7' }, - FLAGGED: { bg: '#e65100', fg: '#ffe0b2' }, - VERIFIED: { bg: '#2e7d32', fg: '#c8e6c9' }, - REJECTED: { bg: '#b71c1c', fg: '#ffcdd2' }, - PENDING: { bg: '#37474f', fg: '#cfd8dc' }, - // service states - HEALTHY: { bg: '#1b5e20', fg: '#a5d6a7' }, - RUNNING: { bg: '#0d47a1', fg: '#90caf9' }, - DEGRADED: { bg: '#f9a825', fg: '#212121' }, - BLOCKED: { bg: '#e65100', fg: '#ffe0b2' }, - FAILED: { bg: '#b71c1c', fg: '#ffcdd2' }, - IDLE: { bg: '#37474f', fg: '#b0bec5' }, - // severities - critical: { bg: '#b71c1c', fg: '#ffcdd2' }, - high: { bg: '#e65100', fg: '#ffe0b2' }, - medium: { bg: '#f9a825', fg: '#212121' }, - low: { bg: '#37474f', fg: '#cfd8dc' }, - info: { bg: '#0d47a1', fg: '#90caf9' }, - warning: { bg: '#e65100', fg: '#ffe0b2' }, - // model regression status - improved: { bg: '#1b5e20', fg: '#a5d6a7' }, - regressed: { bg: '#b71c1c', fg: '#ffcdd2' }, - baseline: { bg: '#37474f', fg: '#cfd8dc' }, - unknown: { bg: '#37474f', fg: '#b0bec5' }, - // review task status - open: { bg: '#e65100', fg: '#ffe0b2' }, - in_review: { bg: '#0d47a1', fg: '#90caf9' }, - resolved: { bg: '#1b5e20', fg: '#a5d6a7' }, - // job status - queued: { bg: '#37474f', fg: '#cfd8dc' }, - running: { bg: '#0d47a1', fg: '#90caf9' }, - completed: { bg: '#1b5e20', fg: '#a5d6a7' }, - failed: { bg: '#b71c1c', fg: '#ffcdd2' }, - stopped: { bg: '#37474f', fg: '#b0bec5' }, -}; - -export function StatusChip({ status, size = 'small' }: { status: string; size?: 'small' | 'medium' }) { - const colors = STATUS_COLORS[status] ?? { bg: '#37474f', fg: '#cfd8dc' }; - return ( - - ); -} - -// ---------------------------------------------------------------- GateLineList - -function fmtGateValue(v: number | string | boolean): string { - if (typeof v === 'number') return v.toLocaleString(undefined, { maximumFractionDigits: 3 }); - return String(v); -} - -export function GateLineList({ checks }: { checks: GateLine[] }) { - if (!checks.length) { - return ( - - No gate results available. - - ); - } - return ( - - - - Gate - Actual - Threshold - Result - - - - {checks.map((c, i) => ( - - {c.gate} - - {fmtGateValue(c.actual)} - - - {fmtGateValue(c.threshold)} - - - {!c.applicable ? ( - - ) : c.passed ? ( - - ) : ( - - )} - - - ))} - -
    - ); -} - -// ---------------------------------------------------------------- layout helpers - -export function SectionCard({ - title, - action, - children, - sx, -}: { - title: ReactNode; - action?: ReactNode; - children: ReactNode; - sx?: object; -}) { - return ( - - - - - {title} - - {action} - - {children} - - - ); -} - -export function LoadingBox({ label = 'Loading…' }: { label?: string }) { - return ( - - - - {label} - - - ); -} - -export function ErrorNote({ error }: { error: string }) { - return ( - - {error.startsWith('API') ? error : `Backend unavailable: ${error}`} - - ); -} - -export function EmptyState({ - title, - message, - action, -}: { - title: string; - message: string; - action?: ReactNode; -}) { - return ( - - - {title} - - - {message} - - {action} - - ); -} - -// ---------------------------------------------------------------- horizontal bar - -export function HBar({ - label, - value, - max, - color = '#4fc3f7', - valueLabel, -}: { - label: string; - value: number; - max: number; - color?: string; - valueLabel?: string; -}) { - const width = max > 0 ? Math.max(1.5, (value / max) * 100) : 0; - return ( - - - - {label} - - - {valueLabel ?? value.toLocaleString()} - - - - - - - ); -} diff --git a/hf-space/src/components/megaeval/CohortTab.tsx b/hf-space/src/components/megaeval/CohortTab.tsx deleted file mode 100644 index 5403c14e8ea42edca28fb18c08df60a504e6c4f3..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/CohortTab.tsx +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Cohort explorer: pick a dimension tab, click a cohort value to drill — - * the value becomes a filter breadcrumb (e.g. Pedestrian → Night → Occluded) - * and the next level groups by the newly selected dimension. Every node shows - * cube metrics via POST /api/evaluations/query. "Why?" runs the factor - * decomposition for the current filter set. - */ -import { useMemo, useState } from 'react'; -import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import Chip from '@mui/material/Chip'; -import CircularProgress from '@mui/material/CircularProgress'; -import Tab from '@mui/material/Tab'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Tabs from '@mui/material/Tabs'; -import Typography from '@mui/material/Typography'; -import { HelpCircle, ChevronRight } from 'lucide-react'; -import type { DimName, WhyResponse } from '../../types/megaeval'; -import { DIM_NAMES } from '../../types/megaeval'; -import { fmtCompact, queryEvaluations, whyAnalysis } from '../../services/megaeval'; -import { usePoll } from '../../services/labeleval'; -import { ErrorNote, HBar, LoadingBox, SectionCard, fmtNum, fmtPct } from '../labeleval/shared'; -import { PctBarCell, QueryBadge, rowNum, rowStr } from './shared'; - -interface Crumb { - dim: DimName; - value: string; -} - -function crumbsToFilters(crumbs: Crumb[]): Partial> { - const filters: Partial> = {}; - for (const c of crumbs) { - filters[c.dim] = [...(filters[c.dim] ?? []), c.value]; - } - return filters; -} - -function WhyPanel({ why }: { why: WhyResponse }) { - const maxCohortN = Math.max(1, ...why.top_cohorts.map((c) => c.n)); - return ( - - - {fmtCompact(why.failure_count)} failures decomposed · {why.method} - - {why.factors.map((f) => ( - - ))} - {why.top_cohorts.length ? ( - <> - - Top affected cohorts - - {why.top_cohorts.map((c) => ( - - ))} - - ) : null} - - ); -} - -export default function CohortTab({ runId, refreshKey }: { runId: string; refreshKey: number }) { - const [crumbs, setCrumbs] = useState([]); - const [dim, setDim] = useState('class'); - const [why, setWhy] = useState(null); - const [whyBusy, setWhyBusy] = useState(false); - const [whyError, setWhyError] = useState(null); - - const filters = useMemo(() => crumbsToFilters(crumbs), [crumbs]); - const filtersKey = JSON.stringify(filters); - - const query = usePoll( - () => - queryEvaluations({ - evaluation_id: runId, - filters, - group_by: [dim], - metrics: ['n', 'recall', 'precision', 'mean_iou', 'anomaly_rate'], - }), - null, - [runId, refreshKey, dim, filtersKey] - ); - - const usedDims = new Set(crumbs.map((c) => c.dim)); - - const drillInto = (value: string) => { - const next = [...crumbs, { dim, value }]; - setCrumbs(next); - setWhy(null); - // Auto-advance the group-by dimension to the next unused one. - const nextUsed = new Set(next.map((c) => c.dim)); - const nextDim = DIM_NAMES.find((d) => !nextUsed.has(d)); - if (nextDim) setDim(nextDim); - }; - - const removeCrumb = (idx: number) => { - setCrumbs((prev) => prev.filter((_, i) => i !== idx)); - setWhy(null); - }; - - const runWhy = async () => { - setWhyBusy(true); - setWhyError(null); - try { - setWhy(await whyAnalysis({ run_id: runId, filters, metric: 'recall' })); - } catch (err) { - setWhyError(err instanceof Error ? err.message : String(err)); - } finally { - setWhyBusy(false); - } - }; - - const rows = useMemo( - () => - query.data - ? // usePoll keeps the previous response while a new query is in flight; - // drop rows that don't carry the currently-selected dimension so stale - // groupings never render (and never collide on duplicate row keys). - query.data.rows - .filter((r) => typeof r[dim] === 'string') - .sort((a, b) => b.n - a.n) - : [], - [query.data, dim] - ); - - return ( - - - - - - } - sx={{ flex: '2 1 560px' }} - > - {/* Filter breadcrumb */} - - { - setCrumbs([]); - setWhy(null); - }} - sx={{ height: 22, bgcolor: crumbs.length ? '#232a31' : '#12303f', color: '#81d4fa', fontWeight: 700 }} - /> - {crumbs.map((c, i) => ( - - - removeCrumb(i)} - sx={{ height: 22, bgcolor: '#12303f', color: '#81d4fa', fontFamily: 'monospace', fontSize: 11 }} - /> - - ))} - - - {/* Dimension tabs */} - setDim(v)} - variant="scrollable" - scrollButtons="auto" - sx={{ minHeight: 32, mb: 1, '& .MuiTab-root': { minHeight: 32, py: 0.5, fontSize: 12 } }} - > - {DIM_NAMES.map((d) => ( - - ))} - - - {query.error ? : null} - {query.loading && !query.data ? : null} - {query.data ? ( - - - - {dim.replace(/_/g, ' ')} - n - Recall - Precision - Mean IoU - Anomaly rate - - - - {rows.map((row, i) => { - const value = rowStr(row, dim); - return ( - drillInto(value)} - sx={{ cursor: 'pointer' }} - > - {value} - - {fmtCompact(row.n)} - - - - - - - - - {fmtNum(rowNum(row, 'mean_iou'), 3)} - - - {fmtPct(rowNum(row, 'anomaly_rate'))} - - - ); - })} - -
    - ) : null} - - Click a row to drill into that cohort; the next level groups by the selected dimension tab. - - - - {whyError ? ( - - - - ) : null} - {why ? : null} -     - ); -} diff --git a/hf-space/src/components/megaeval/CompareTab.tsx b/hf-space/src/components/megaeval/CompareTab.tsx deleted file mode 100644 index 773c7c4b6821be7ed31371e1a7caea3ed1e4beb0..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/CompareTab.tsx +++ /dev/null @@ -1,221 +0,0 @@ -/** - * Model-vs-model comparison: candidate (selected run) against the baseline - * chosen in the header. Headline deltas, per-class deltas, top regressions, - * and the PROMOTE / DO_NOT_PROMOTE recommendation with blockers. - */ -import Alert from '@mui/material/Alert'; -import AlertTitle from '@mui/material/AlertTitle'; -import Box from '@mui/material/Box'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import { compareRuns, fmtCompact } from '../../services/megaeval'; -import { usePoll } from '../../services/labeleval'; -import { ErrorNote, LoadingBox, SectionCard, fmtPct } from '../labeleval/shared'; -import { DeltaText } from './shared'; - -/** Metrics where a positive delta is a regression rather than an improvement. */ -const LOWER_IS_BETTER = new Set(['anomaly_rate', 'error_rate', 'fp', 'fn', 'loc_err', 'anomalies']); - -function fmtMetricValue(v: number): string { - // Rates arrive as fractions; counts as integers > 1. - return Math.abs(v) <= 1 ? fmtPct(v) : fmtCompact(v); -} - -export default function CompareTab({ - runId, - baselineRunId, - refreshKey, -}: { - runId: string; - baselineRunId: string | null; - refreshKey: number; -}) { - const cmp = usePoll( - () => - baselineRunId - ? compareRuns({ candidate_run_id: runId, baseline_run_id: baselineRunId }) - : Promise.resolve(null), - null, - [runId, baselineRunId, refreshKey] - ); - - if (!baselineRunId) { - return ( - - Select a baseline run in the header to compare the current candidate against it. - - ); - } - if (cmp.error) return ; - if (cmp.loading && !cmp.data) return ; - const data = cmp.data; - if (!data) return null; - - const promote = data.recommendation === 'PROMOTE'; - - return ( - - - - {promote ? 'PROMOTE' : 'DO NOT PROMOTE'} — {data.candidate.model_version} vs baseline{' '} - {data.baseline.model_version} - - {data.blockers.length ? ( - - {data.blockers.map((b, i) => ( -
  • - {b} -
    • - ))} -     - ) : ( - No blockers under the current promotion policy. - )} -     - - - - - - - Metric - Baseline - Candidate - Δ - - - - {data.headline_deltas.map((d) => ( - - {d.metric} - - {fmtMetricValue(d.baseline)} - - - {fmtMetricValue(d.candidate)} - - - - - - ))} - -
    -     - - - - - - Class - n - Recall - Δ recall - Precision - Δ precision - - - - {data.per_class.map((c) => ( - - {c.class} - - {fmtCompact(c.n)} - - - {fmtPct(c.recall_baseline)} → {fmtPct(c.recall_candidate)} - - - - - - {fmtPct(c.precision_baseline)} → {fmtPct(c.precision_candidate)} - - - - - - ))} - -
    -     -     - - - {data.regressions.length === 0 ? ( - - No cohort regressions detected against the baseline. - - ) : ( - - {data.regressions.map((r) => ( - - - 🚨 REGRESSION: - - - {r.cohort} recall {fmtPct(r.recall_baseline)} → {fmtPct(r.recall_candidate)} (Δ{' '} - ) - - - n={fmtCompact(r.n)} - - - ))} - - )} - - - {data.worst_cohorts.length ? ( - - - - - Cohort - n - Recall (baseline → candidate) - Δ - - - - {data.worst_cohorts.map((c) => ( - - {c.cohort} - - {fmtCompact(c.n)} - - - {fmtPct(c.recall_baseline)} → {fmtPct(c.recall_candidate)} - - - - - - ))} - -
    -     - ) : null} -     - ); -} diff --git a/hf-space/src/components/megaeval/ContainersTab.tsx b/hf-space/src/components/megaeval/ContainersTab.tsx deleted file mode 100644 index 2b08722938025a54f82c807a4214788a9fee9851..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/ContainersTab.tsx +++ /dev/null @@ -1,321 +0,0 @@ -/** - * Container explorer ("Quality by Container"): sortable container table with - * risk presets. Drilling into a container opens the forensic object table — - * intentionally the deepest level of the app — plus nearest-neighbor - * "similar containers" retrieval for casting a wider net. - */ -import { useState } from 'react'; -import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import CircularProgress from '@mui/material/CircularProgress'; -import MenuItem from '@mui/material/MenuItem'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import TextField from '@mui/material/TextField'; -import Typography from '@mui/material/Typography'; -import { ArrowLeft, Search, ShieldAlert } from 'lucide-react'; -import type { ContainerSortPreset, SimilarityResponse } from '../../types/megaeval'; -import { - findSimilarContainers, - fmtCompact, - getContainerObjects, - getRunContainers, -} from '../../services/megaeval'; -import { usePoll } from '../../services/labeleval'; -import { ErrorNote, LoadingBox, SectionCard, fmtNum, fmtPct } from '../labeleval/shared'; -import { ContainerStatusIcon, DimChips, OutcomeChip } from './shared'; - -const SORT_PRESETS: Array<{ value: ContainerSortPreset; label: string }> = [ - { value: 'highest_risk', label: 'Highest risk' }, - { value: 'worst_recall', label: 'Worst recall' }, - { value: 'worst_precision', label: 'Worst precision' }, - { value: 'worst_iou', label: 'Worst IoU' }, - { value: 'most_anomalies', label: 'Most anomalies' }, - { value: 'least_verified', label: 'Least verified' }, -]; - -// ---------------------------------------------------------------- forensic view - -function ForensicView({ - runId, - containerId, - onBack, - onDrill, -}: { - runId: string; - containerId: number; - onBack: () => void; - onDrill: (cid: number) => void; -}) { - const objects = usePoll(() => getContainerObjects(runId, containerId), null, [runId, containerId]); - const [similar, setSimilar] = useState(null); - const [simBusy, setSimBusy] = useState(false); - const [simError, setSimError] = useState(null); - - const findSimilar = async () => { - setSimBusy(true); - setSimError(null); - try { - setSimilar(await findSimilarContainers({ run_id: runId, container_id: containerId, k: 10 })); - } catch (err) { - setSimError(err instanceof Error ? err.message : String(err)); - } finally { - setSimBusy(false); - } - }; - - return ( - - - - Forensic object table · container #{containerId} - - } - action={ - - } - sx={{ flex: '2 1 620px' }} - > - - Deepest drill-down level: individual annotations, loaded on demand for this container only. - - {objects.error ? : null} - {objects.loading && !objects.data ? : null} - {objects.data ? ( - - - - - Outcome - Class - IoU - Conf - Anomaly - Dims - - - - {objects.data.objects.map((o) => ( - - - - - {o.safety_critical ? : null} - - - {o.class} - - {fmtNum(o.iou, 3)} - - - {fmtNum(o.confidence, 2)} - - - {o.anomaly ? ( - - ● - - ) : ( - - ○ - - )} - - - - - - ))} - -
    -     - ) : null} - - - {simError ? ( - - - - ) : null} - {similar ? ( - - - - - Container - Similarity - Context - FN/FP - Risk - - - - {similar.results.map((s) => ( - onDrill(s.container_id)} sx={{ cursor: 'pointer' }}> - #{s.container_id} - - {fmtPct(s.similarity)} - - - - - - {s.fn}/{s.fp} - - - {fmtNum(s.risk_score, 2)} - - - ))} - -
    -     - ) : null} -     - ); -} - -// ---------------------------------------------------------------- containers table - -export default function ContainersTab({ - runId, - refreshKey, - drillContainerId, - onDrill, -}: { - runId: string; - refreshKey: number; - drillContainerId: number | null; - onDrill: (cid: number | null) => void; -}) { - const [sort, setSort] = useState('highest_risk'); - const containers = usePoll(() => getRunContainers(runId, sort, 50), null, [runId, sort, refreshKey]); - - if (drillContainerId !== null) { - return ( - onDrill(null)} - onDrill={(cid) => onDrill(cid)} - /> - ); - } - - return ( - - {containers.data ? ( - - {fmtCompact(containers.data.total)} containers (exact) · showing top {containers.data.rows.length} - - ) : null} - setSort(e.target.value as ContainerSortPreset)} - sx={{ minWidth: 180 }} - > - {SORT_PRESETS.map((p) => ( - - {p.label} - - ))} - - - } - > - {containers.error ? : null} - {containers.loading && !containers.data ? : null} - {containers.data ? ( - - - - - - Container - Context - Objects - Verified - Precision - Recall - Mean IoU - Anomalies - Risk - - - - {containers.data.rows.map((c) => ( - onDrill(c.container_id)} sx={{ cursor: 'pointer' }}> - - - - #{c.container_id} - - - - - {fmtCompact(c.n_objects)} - - - {c.verified} - - - {fmtPct(c.precision)} - - - {fmtPct(c.recall)} - - - {fmtNum(c.mean_iou, 3)} - - 0 ? '#ec407a' : undefined }}> - {c.anomalies} - - - {fmtNum(c.risk_score, 2)} - - - ))} - -
    -     - ) : null} - - Click a container to open its forensic object table (deepest drill-down). - -     - ); -} diff --git a/hf-space/src/components/megaeval/HeaderBar.tsx b/hf-space/src/components/megaeval/HeaderBar.tsx deleted file mode 100644 index 575adc11b061ca0f541097c9ce31d86af5a8d69c..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/HeaderBar.tsx +++ /dev/null @@ -1,335 +0,0 @@ -/** - * Command Center header: population + run + baseline selectors, run status, - * cache stats chip, and the two entry-point actions (generate population, - * launch evaluation run). The action buttons are exported so the page-level - * empty state can reuse them front and center. - */ -import { useMemo, useState } from 'react'; -import Alert from '@mui/material/Alert'; -import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import Chip from '@mui/material/Chip'; -import CircularProgress from '@mui/material/CircularProgress'; -import Dialog from '@mui/material/Dialog'; -import DialogActions from '@mui/material/DialogActions'; -import DialogContent from '@mui/material/DialogContent'; -import DialogTitle from '@mui/material/DialogTitle'; -import FormControlLabel from '@mui/material/FormControlLabel'; -import MenuItem from '@mui/material/MenuItem'; -import Switch from '@mui/material/Switch'; -import TextField from '@mui/material/TextField'; -import Tooltip from '@mui/material/Tooltip'; -import Typography from '@mui/material/Typography'; -import { Database, FlaskConical } from 'lucide-react'; -import type { CacheStats, EvaluationRunInfo, PopulationMeta } from '../../types/megaeval'; -import { createRun, fmtCompact, generatePopulation } from '../../services/megaeval'; -import { fmtPct } from '../labeleval/shared'; -import { RunStatusChip } from './shared'; - -// ---------------------------------------------------------------- action buttons - -export function GeneratePopulationButton({ - onGenerated, - variant = 'outlined', - size = 'small', -}: { - onGenerated: (pop: PopulationMeta) => void; - variant?: 'outlined' | 'contained'; - size?: 'small' | 'medium' | 'large'; -}) { - const [open, setOpen] = useState(false); - const [numObjects, setNumObjects] = useState('320000'); - const [name, setName] = useState(''); - const [busy, setBusy] = useState(false); - const [error, setError] = useState(null); - - const submit = async () => { - setBusy(true); - setError(null); - try { - const parsed = Number(numObjects); - const pop = await generatePopulation({ - ...(name.trim() ? { name: name.trim() } : {}), - num_objects: Number.isFinite(parsed) && parsed > 0 ? Math.round(parsed) : 320000, - }); - setOpen(false); - onGenerated(pop); - } catch (err) { - setError(err instanceof Error ? err.message : String(err)); - } finally { - setBusy(false); - } - }; - - return ( - <> - - (busy ? undefined : setOpen(false))} maxWidth="xs" fullWidth> - Generate evaluation population - - {error ? ( - - {error} - - ) : null} - setName(e.target.value)} - placeholder="e.g. q3-highway-mix" - /> - setNumObjects(e.target.value)} - helperText="1K – 1.2M synthetic annotated objects (default 320K)" - /> - - - - - - - - ); -} - -export function NewRunButton({ - populationId, - onCreated, - variant = 'contained', - size = 'small', -}: { - populationId: string | null; - onCreated: (run: EvaluationRunInfo) => void; - variant?: 'outlined' | 'contained'; - size?: 'small' | 'medium' | 'large'; -}) { - const [open, setOpen] = useState(false); - const [modelVersion, setModelVersion] = useState('perception-v2.1'); - const [injectRegression, setInjectRegression] = useState(false); - const [busy, setBusy] = useState(false); - const [error, setError] = useState(null); - - const submit = async () => { - if (!populationId) return; - setBusy(true); - setError(null); - try { - const run = await createRun({ - population_id: populationId, - model_version: modelVersion.trim() || 'perception-v2.1', - ...(injectRegression ? { overrides: { night_penalty: 0.3, vru_penalty: 0.1 } } : {}), - }); - setOpen(false); - onCreated(run); - } catch (err) { - setError(err instanceof Error ? err.message : String(err)); - } finally { - setBusy(false); - } - }; - - return ( - <> - - - - - - (busy ? undefined : setOpen(false))} maxWidth="xs" fullWidth> - Launch evaluation run - - {error ? ( - - {error} - - ) : null} - setModelVersion(e.target.value)} - /> - setInjectRegression(e.target.checked)} /> - } - label={ - - Inject regression (demo) - - Applies overrides night_penalty=0.30, vru_penalty=0.10 - - - } - /> - - Runs are asynchronous: the run is queued, evaluated across distributed workers, then published. - - - - - - - - - ); -} - -// ---------------------------------------------------------------- header bar - -export default function HeaderBar({ - populations, - populationId, - onPopulationChange, - runs, - runId, - onRunChange, - baselineRunId, - onBaselineChange, - selectedRun, - cache, - onGenerated, - onRunCreated, -}: { - populations: PopulationMeta[]; - populationId: string | null; - onPopulationChange: (id: string) => void; - runs: EvaluationRunInfo[]; - runId: string | null; - onRunChange: (id: string) => void; - baselineRunId: string | null; - onBaselineChange: (id: string | null) => void; - selectedRun: EvaluationRunInfo | null; - cache: CacheStats | null; - onGenerated: (pop: PopulationMeta) => void; - onRunCreated: (run: EvaluationRunInfo) => void; -}) { - const sortedRuns = useMemo( - () => [...runs].sort((a, b) => (b.created_at ?? '').localeCompare(a.created_at ?? '')), - [runs] - ); - const publishedRuns = useMemo(() => sortedRuns.filter((r) => r.status === 'published'), [sortedRuns]); - - const runLabel = (r: EvaluationRunInfo) => - `${r.model_version} · ${r.run_id.slice(0, 8)}${r.status === 'published' ? '' : ` (${r.status})`}`; - - return ( - - onPopulationChange(e.target.value)} - sx={{ minWidth: 220 }} - disabled={populations.length === 0} - > - {populations.map((p) => ( - - {p.name} · {fmtCompact(p.num_objects)} objects - - ))} - - - r.run_id === runId) ? runId : ''} - onChange={(e) => onRunChange(e.target.value)} - sx={{ minWidth: 240 }} - disabled={sortedRuns.length === 0} - > - {sortedRuns.map((r) => ( - - {runLabel(r)} - - ))} - - - r.run_id === baselineRunId) ? baselineRunId : '' - } - onChange={(e) => onBaselineChange(e.target.value === '' ? null : e.target.value)} - sx={{ minWidth: 220 }} - disabled={publishedRuns.length === 0} - > - - None - - {publishedRuns - .filter((r) => r.run_id !== runId) - .map((r) => ( - - {runLabel(r)} - - ))} - - - {selectedRun ? : null} - - - - {cache ? ( - - - - ) : null} - - - - - ); -} diff --git a/hf-space/src/components/megaeval/HeroRow.tsx b/hf-space/src/components/megaeval/HeroRow.tsx deleted file mode 100644 index 7226abbd419babec85423331f38307c498943821..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/HeroRow.tsx +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Hero aggregates for a published run: population-scale counts followed by the - * headline quality metrics. Precision/recall are annotated with sampling-verified - * confidence intervals when review results exist. - */ -import Box from '@mui/material/Box'; -import Chip from '@mui/material/Chip'; -import type { EvaluationRunInfo, MetricEstimate, ReviewState } from '../../types/megaeval'; -import { fmtCompact, queryEvaluations } from '../../services/megaeval'; -import { usePoll } from '../../services/labeleval'; -import { MetricCard, fmtNum, fmtPct } from '../labeleval/shared'; -import { QueryBadge, rowNum } from './shared'; - -function ciSub(est: MetricEstimate | undefined): string | null { - if (!est) return null; - return `95% CI ${fmtPct(est.ci_low)}–${fmtPct(est.ci_high)}, n=${est.n_reviewed.toLocaleString()} reviewed`; -} - -const VERIFIED_CHIP = ( - -); - -export default function HeroRow({ - run, - review, - refreshKey, -}: { - run: EvaluationRunInfo; - review: ReviewState | null; - refreshKey: number; -}) { - const h = run.headline; - // Reviewed / verified totals come from the aggregate query API (also shows provenance). - const totals = usePoll( - () => - queryEvaluations({ - evaluation_id: run.run_id, - metrics: ['n', 'reviewed', 'verified', 'review_coverage'], - }), - null, - [run.run_id, refreshKey] - ); - const totalsRow = totals.data?.rows[0]; - const reviewed = totalsRow ? rowNum(totalsRow, 'reviewed') : null; - const verified = totalsRow ? rowNum(totalsRow, 'verified') : null; - - const results = review?.results ?? null; - const precisionCi = ciSub(results?.precision); - const recallCi = ciSub(results?.recall); - - return ( - - - - - - human reviews - - - } - accent="#42a5f5" - /> - - - - - - {fmtPct(results?.precision.estimate)} ({precisionCi}) - - {VERIFIED_CHIP} - - ) : ( - 'point estimate' - ) - } - /> - - - {fmtPct(results?.recall.estimate)} ({recallCi}) - - {VERIFIED_CHIP} - - ) : ( - 'point estimate' - ) - } - /> - - - - - - - ); -} diff --git a/hf-space/src/components/megaeval/InvestigationTab.tsx b/hf-space/src/components/megaeval/InvestigationTab.tsx deleted file mode 100644 index 5ea5ed18849db0a63e2300ed9a4ccc2565c34dab..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/InvestigationTab.tsx +++ /dev/null @@ -1,266 +0,0 @@ -/** - * Investigation panel: multi-criteria error search against the inverted error - * index. Results stay aggregate-first (counts, by-type shares, worst - * containers) — clicking a container jumps to the forensic drill-down. - */ -import { useState } from 'react'; -import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import Checkbox from '@mui/material/Checkbox'; -import Chip from '@mui/material/Chip'; -import CircularProgress from '@mui/material/CircularProgress'; -import FormControlLabel from '@mui/material/FormControlLabel'; -import MenuItem from '@mui/material/MenuItem'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import TextField from '@mui/material/TextField'; -import Typography from '@mui/material/Typography'; -import { SearchCode, ShieldAlert } from 'lucide-react'; -import type { DimName, ErrorSearchResponse, ErrorType } from '../../types/megaeval'; -import { fmtCompact, getDimensions, searchErrors } from '../../services/megaeval'; -import { usePoll } from '../../services/labeleval'; -import { ErrorNote, MetricCard, SectionCard, fmtNum } from '../labeleval/shared'; -import { DimChips, ERROR_TYPE_COLORS, OutcomeChip } from './shared'; - -const ALL_ERROR_TYPES: ErrorType[] = ['FN', 'FP', 'LOCALIZATION', 'ANOMALY', 'LOW_CONF']; - -function MultiSelect({ - label, - options, - value, - onChange, - minWidth = 170, -}: { - label: string; - options: string[]; - value: string[]; - onChange: (v: string[]) => void; - minWidth?: number; -}) { - return ( - { - const v = e.target.value as unknown; - onChange(typeof v === 'string' ? v.split(',').filter(Boolean) : (v as string[])); - }} - SelectProps={{ multiple: true }} - sx={{ minWidth }} - > - {options.map((o) => ( - - {o} - - ))} - - ); -} - -export default function InvestigationTab({ - runId, - onOpenContainer, -}: { - runId: string; - onOpenContainer: (cid: number) => void; -}) { - const dims = usePoll(getDimensions, null, []); - const dimOptions = (d: DimName): string[] => dims.data?.dimensions[d] ?? []; - - const [errorTypes, setErrorTypes] = useState([]); - const [classes, setClasses] = useState([]); - const [lightings, setLightings] = useState([]); - const [weathers, setWeathers] = useState([]); - const [confMax, setConfMax] = useState(''); - const [riskMin, setRiskMin] = useState(''); - const [safetyOnly, setSafetyOnly] = useState(false); - - const [result, setResult] = useState(null); - const [busy, setBusy] = useState(false); - const [error, setError] = useState(null); - - const runSearch = async () => { - setBusy(true); - setError(null); - try { - const filters: Partial> = {}; - if (classes.length) filters.class = classes; - if (lightings.length) filters.lighting = lightings; - if (weathers.length) filters.weather = weathers; - const confParsed = Number(confMax); - const riskParsed = Number(riskMin); - setResult( - await searchErrors({ - run_id: runId, - ...(errorTypes.length ? { error_types: errorTypes as ErrorType[] } : {}), - ...(Object.keys(filters).length ? { filters } : {}), - ...(confMax.trim() !== '' && Number.isFinite(confParsed) ? { confidence_max: confParsed } : {}), - ...(riskMin.trim() !== '' && Number.isFinite(riskParsed) ? { risk_min: riskParsed } : {}), - ...(safetyOnly ? { safety_only: true } : {}), - limit_containers: 20, - }) - ); - } catch (err) { - setError(err instanceof Error ? err.message : String(err)); - } finally { - setBusy(false); - } - }; - - return ( - - - - - - - - setConfMax(e.target.value)} - sx={{ width: 130 }} - inputProps={{ step: 0.05, min: 0, max: 1 }} - /> - setRiskMin(e.target.value)} - sx={{ width: 110 }} - inputProps={{ step: 0.1, min: 0 }} - /> - setSafetyOnly(e.target.checked)} />} - label={Safety-critical only} - /> - - - {error ? : null} - - - {result ? ( - <> - - - - {ALL_ERROR_TYPES.filter((t) => (result.by_type[t] ?? 0) > 0).map((t) => ( - - ))} - - - - - - - - - Container - Context - Errors - Mean risk - Max severity - Safety hits - - - - {result.worst_containers.map((c) => ( - onOpenContainer(c.container_id)} - sx={{ cursor: 'pointer' }} - > - #{c.container_id} - - - - - {c.error_count} - - - {fmtNum(c.mean_risk, 2)} - - - {fmtNum(c.max_severity, 2)} - - 0 ? '#ef5350' : undefined }}> - {c.safety_hits} - - - ))} - -
    - - Click a container to open its forensic object table. - -     - - - - {result.examples.map((ex) => ( - onOpenContainer(ex.container_id)} - > - - {ex.safety_critical ? : null} - - {ex.class} - - - - - sev {fmtNum(ex.severity, 2)} · conf {fmtNum(ex.confidence, 2)} · risk {fmtNum(ex.risk_score, 2)} · # - {ex.container_id} - - - ))} - - -     - - ) : ( - - Define criteria and search the error index — results are served from the inverted index, not a scan. - - )} -     - ); -} diff --git a/hf-space/src/components/megaeval/LineageCard.tsx b/hf-space/src/components/megaeval/LineageCard.tsx deleted file mode 100644 index 71364ede059f51ea17331913e42d7c2e00aef7da..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/LineageCard.tsx +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Collapsible reproducibility card: every version pin that produced this - * evaluation run (dataset, model, labels, evaluator code, metric definitions, - * thresholds, sampling config, seed, hardware). - */ -import { useState } from 'react'; -import Box from '@mui/material/Box'; -import Collapse from '@mui/material/Collapse'; -import IconButton from '@mui/material/IconButton'; -import Typography from '@mui/material/Typography'; -import { ChevronDown, ChevronUp } from 'lucide-react'; -import type { RunLineage } from '../../types/megaeval'; -import { SectionCard } from '../labeleval/shared'; - -function Field({ label, value }: { label: string; value: string | number }) { - return ( - - - {label} - - - {value} - - - ); -} - -export default function LineageCard({ lineage }: { lineage: RunLineage }) { - const [open, setOpen] = useState(true); - return ( - setOpen((o) => !o)}> - {open ? : } - - } - > - - - - - - - - - - - - - - - - - Threshold config - - - {JSON.stringify(lineage.threshold_config, null, 2)} - - - - - Sampling config - - - {JSON.stringify(lineage.sampling_config, null, 2)} - - - - - Identical inputs + seed reproduce this evaluation bit-for-bit. - - - - ); -} diff --git a/hf-space/src/components/megaeval/QualityTab.tsx b/hf-space/src/components/megaeval/QualityTab.tsx deleted file mode 100644 index 19d2020b82aefb0f785311b54cbb92811b379df1..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/QualityTab.tsx +++ /dev/null @@ -1,358 +0,0 @@ -/** - * Quality tab: per-class quality bars, error-type distribution, the quality - * funnel, cross-run quality trend, and sketch-based distributions. - * Every aggregate-query panel surfaces its provenance badge. - */ -import { useMemo } from 'react'; -import Box from '@mui/material/Box'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import type { ErrorType, EvaluationRunInfo } from '../../types/megaeval'; -import { - fmtCompact, - getRunDistributions, - getRunFunnel, - queryEvaluations, - searchErrors, -} from '../../services/megaeval'; -import { usePoll } from '../../services/labeleval'; -import { ErrorNote, HBar, LoadingBox, SectionCard, fmtNum, fmtPct } from '../labeleval/shared'; -import { - ERROR_TYPE_COLORS, - ExactnessTag, - HistogramSparkline, - PctBarCell, - QueryBadge, - rowNum, - rowStr, -} from './shared'; - -// ---------------------------------------------------------------- per-class quality - -function QualityByClass({ runId, refreshKey }: { runId: string; refreshKey: number }) { - const q = usePoll( - () => - queryEvaluations({ - evaluation_id: runId, - group_by: ['class'], - metrics: ['n', 'precision', 'recall', 'f1', 'mean_iou'], - }), - null, - [runId, refreshKey] - ); - return ( - } sx={{ flex: '1 1 460px' }}> - {q.error ? : null} - {q.loading && !q.data ? : null} - {q.data ? ( - - - - Class - n - Recall - Precision - F1 - Mean IoU - - - - {[...q.data.rows] - .sort((a, b) => b.n - a.n) - .map((row) => ( - - {rowStr(row, 'class')} - - {fmtCompact(row.n)} - - - - - - - - - {fmtPct(rowNum(row, 'f1'))} - - - {fmtNum(rowNum(row, 'mean_iou'), 3)} - - - ))} - -
    - ) : null} -     - ); -} - -// ---------------------------------------------------------------- error distribution - -const ERROR_TYPE_ORDER: ErrorType[] = ['FN', 'FP', 'LOCALIZATION', 'ANOMALY', 'LOW_CONF']; - -function ErrorDistribution({ runId, refreshKey }: { runId: string; refreshKey: number }) { - const res = usePoll(() => searchErrors({ run_id: runId }), null, [runId, refreshKey]); - const byType = res.data?.by_type ?? {}; - const total = ERROR_TYPE_ORDER.reduce((acc, t) => acc + (byType[t] ?? 0), 0); - const max = Math.max(1, ...ERROR_TYPE_ORDER.map((t) => byType[t] ?? 0)); - return ( - - {res.error ? : null} - {res.loading && !res.data ? : null} - {res.data ? ( - <> - - {fmtCompact(res.data.matched_errors)} potential errors indexed (exact) - - {ERROR_TYPE_ORDER.map((t) => { - const count = byType[t] ?? 0; - return ( - 0 ? fmtPct(count / total) : '—'})`} - /> - ); - })} - - ) : null} - - ); -} - -// ---------------------------------------------------------------- quality funnel - -function QualityFunnel({ runId, refreshKey }: { runId: string; refreshKey: number }) { - const funnel = usePoll(() => getRunFunnel(runId), null, [runId, refreshKey]); - const stages = funnel.data?.stages ?? []; - const max = stages.length ? Math.max(...stages.map((s) => s.count)) : 1; - const est = funnel.data?.estimated_precision ?? null; - return ( - - {funnel.error ? : null} - {funnel.loading && !funnel.data ? : null} - {funnel.data ? ( - <> - {stages.map((s) => ( - - ))} - {est && est.estimate !== null ? ( - - Estimated precision {fmtPct(est.estimate)} - {est.ci_low !== null && est.ci_high !== null - ? ` (95% CI ${fmtPct(est.ci_low)}–${fmtPct(est.ci_high)}${ - est.n_reviewed !== null ? `, n=${est.n_reviewed.toLocaleString()} reviewed` : '' - })` - : ''} - - ) : null} - - ) : null} - - ); -} - -// ---------------------------------------------------------------- quality trend - -const TREND_SERIES: Array<{ key: 'precision' | 'recall' | 'safety_recall'; label: string; color: string }> = [ - { key: 'precision', label: 'precision', color: '#4fc3f7' }, - { key: 'recall', label: 'recall', color: '#66bb6a' }, - { key: 'safety_recall', label: 'safety recall', color: '#ffa726' }, -]; - -function QualityTrend({ runs }: { runs: EvaluationRunInfo[] }) { - const published = useMemo( - () => - runs - .filter((r) => r.status === 'published') - .sort((a, b) => (a.created_at ?? '').localeCompare(b.created_at ?? '')), - [runs] - ); - - const width = 520; - const height = 150; - const padL = 40; - const padB = 30; - const padT = 10; - - const values = published.flatMap((r) => - TREND_SERIES.map((s) => r.headline[s.key]).filter((v): v is number => v !== null && v !== undefined) - ); - const lo = values.length ? Math.max(0, Math.min(...values) - 0.05) : 0; - const hi = values.length ? Math.min(1, Math.max(...values) + 0.02) : 1; - const xFor = (i: number) => - padL + (published.length <= 1 ? (width - padL) / 2 : (i / (published.length - 1)) * (width - padL - 10)); - const yFor = (v: number) => padT + (1 - (v - lo) / Math.max(1e-9, hi - lo)) * (height - padT - padB); - - return ( - - {published.length < 2 ? ( - - {published.length === 0 - ? 'No published runs yet.' - : 'Only one published run — launch another run to see the trend.'} - - ) : null} - {published.length >= 1 ? ( - <> - - {[lo, (lo + hi) / 2, hi].map((v, i) => ( - - - - {fmtPct(v)} - - - ))} - {TREND_SERIES.map((s) => { - const pts = published - .map((r, i) => ({ v: r.headline[s.key], i })) - .filter((p): p is { v: number; i: number } => p.v !== null && p.v !== undefined); - if (!pts.length) return null; - return ( - - `${xFor(p.i)},${yFor(p.v)}`).join(' ')} - /> - {pts.map((p) => ( - - ))} - - ); - })} - {published.map((r, i) => ( - - {r.model_version.length > 14 ? `${r.model_version.slice(0, 13)}…` : r.model_version} - - ))} - - - {TREND_SERIES.map((s) => ( - - - - {s.label} - - - ))} - - - ) : null} - - ); -} - -// ---------------------------------------------------------------- distributions (sketches) - -function DistributionsCard({ runId, refreshKey }: { runId: string; refreshKey: number }) { - const dist = usePoll(() => getRunDistributions(runId), null, [runId, refreshKey]); - const d = dist.data; - return ( - - Distributions - - - } - sx={{ flex: '1 1 320px' }} - > - {dist.error ? : null} - {dist.loading && !d ? : null} - {d ? ( - - {d.confidence ? ( - - - Confidence — approximate (sketch) - - - - ) : null} - {d.iou ? ( - - - IoU — approximate (sketch) - - - - ) : null} - - - - - Containers (HLL) - - - - - {d.containers_hll_estimate === null ? '—' : fmtCompact(d.containers_hll_estimate)} - - - - - - Containers (exact) - - - - - {d.containers_exact === null ? '—' : fmtCompact(d.containers_exact)} - - - - - {d.note} - - - ) : null} - - ); -} - -// ---------------------------------------------------------------- tab - -export default function QualityTab({ - runId, - runs, - refreshKey, -}: { - runId: string; - runs: EvaluationRunInfo[]; - refreshKey: number; -}) { - return ( - - - - - - - - ); -} diff --git a/hf-space/src/components/megaeval/ReviewTab.tsx b/hf-space/src/components/megaeval/ReviewTab.tsx deleted file mode 100644 index 8ee5c4749faa20168894d6b55427b024fb0ade33..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/ReviewTab.tsx +++ /dev/null @@ -1,205 +0,0 @@ -/** - * Review sampling panel: statistically grounded human review. Build a - * stratified sampling plan, execute (simulated) reviews, then read precision - * and recall estimates with Wilson confidence intervals per stratum. - */ -import { useState } from 'react'; -import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import CircularProgress from '@mui/material/CircularProgress'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import { ClipboardList, UserCheck } from 'lucide-react'; -import type { MetricEstimate, ReviewState, SamplingFunnel, StratumPlan } from '../../types/megaeval'; -import { executeReview, fmtCompact, planReview } from '../../services/megaeval'; -import { ErrorNote, HBar, SectionCard, fmtPct } from '../labeleval/shared'; - -const FUNNEL_STAGES: Array<{ key: keyof SamplingFunnel; label: string }> = [ - { key: 'population_objects', label: 'Population objects' }, - { key: 'containers', label: 'Containers' }, - { key: 'suspicious_containers', label: 'Suspicious containers' }, - { key: 'candidate_pool', label: 'Candidate pool' }, - { key: 'statistically_selected', label: 'Statistically selected' }, - { key: 'reviewed', label: 'Reviewed' }, -]; - -function EstimateCard({ label, est }: { label: string; est: MetricEstimate }) { - return ( - - - {fmtPct(est.estimate)} - - 95% CI {fmtPct(est.ci_low)}–{fmtPct(est.ci_high)} · n={est.n_reviewed.toLocaleString()} reviewed · frame{' '} - {fmtCompact(est.frame_size)} - - - - {est.method} - - - - - Stratum - N - n - p - Wilson 95% CI - - - - {est.strata.map((s) => ( - - {s.stratum} - - {fmtCompact(s.N)} - - - {s.n.toLocaleString()} - - - {fmtPct(s.p)} - - - {fmtPct(s.wilson_ci[0])}–{fmtPct(s.wilson_ci[1])} - - - ))} - -
    -     - ); -} - -function PlanTable({ title, strata }: { title: string; strata: Record }) { - const entries = Object.entries(strata); - return ( - - - - - Stratum - N - Allocated - - - - {entries.map(([key, plan]) => ( - - {plan.label || key} - - {fmtCompact(plan.N)} - - - {plan.allocated.toLocaleString()} - - - ))} - -
    -     - ); -} - -export default function ReviewTab({ - runId, - review, - onReview, -}: { - runId: string; - review: ReviewState | null; - onReview: (r: ReviewState) => void; -}) { - const [busy, setBusy] = useState<'plan' | 'execute' | null>(null); - const [error, setError] = useState(null); - - const act = async (kind: 'plan' | 'execute') => { - setBusy(kind); - setError(null); - try { - onReview(kind === 'plan' ? await planReview(runId) : await executeReview(runId)); - } catch (err) { - setError(err instanceof Error ? err.message : String(err)); - } finally { - setBusy(null); - } - }; - - const funnel = review?.funnel; - const max = funnel ? Math.max(1, funnel.population_objects) : 1; - - return ( - - - - - - } - > - {error ? : null} - {!review || !review.planned ? ( - - No sampling plan yet. Reviewing everything at this scale is impossible — build a stratified plan to - estimate precision and recall from a small, statistically selected sample. - - ) : ( - <> - - {review.method ?? 'stratified sampling'} - {review.target_n !== undefined ? ` · target n=${review.target_n.toLocaleString()}` : ''} - {review.executed ? ' · executed' : ' · planned, not executed'} - - {funnel - ? FUNNEL_STAGES.map((s) => ( - - )) - : null} - - )} - - - {review?.planned && !review.executed && review.strata ? ( - - - - - ) : null} - - {review?.results ? ( - - - - - ) : null} - - ); -} diff --git a/hf-space/src/components/megaeval/RunProgressCard.tsx b/hf-space/src/components/megaeval/RunProgressCard.tsx deleted file mode 100644 index 35a80bf3480973ba91b04b17460fea4d3a642b90..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/RunProgressCard.tsx +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Live progress card for an in-flight (non-published) evaluation run. - * Fed by the SSE stream when available, otherwise by 1.5s polling. - */ -import Box from '@mui/material/Box'; -import LinearProgress from '@mui/material/LinearProgress'; -import Typography from '@mui/material/Typography'; -import type { RunProgress } from '../../types/megaeval'; -import { fmtCompact } from '../../services/megaeval'; -import { SectionCard } from '../labeleval/shared'; -import { CompactStat, RunStatusChip } from './shared'; - -function fmtEta(s: number | null): string { - if (s === null || Number.isNaN(s)) return '—'; - if (s < 60) return `${Math.max(0, Math.round(s))}s`; - return `${Math.floor(s / 60)}m ${Math.round(s % 60)}s`; -} - -export default function RunProgressCard({ progress }: { progress: RunProgress }) { - // Tolerate either 0..1 or 0..100 percent encodings from the backend. - const pct = Math.max(0, Math.min(100, progress.percent <= 1 ? progress.percent * 100 : progress.percent)); - return ( - - - Evaluation run {progress.run_id.slice(0, 8)} · {progress.model_version} - - - - } - > - - - - - - - - - - {progress.error ? ( - - {progress.error} - - ) : ( - - Map partitions across workers → reduce partial aggregates → materialize cube & indexes → publish. - - )} - - ); -} diff --git a/hf-space/src/components/megaeval/ShiftTab.tsx b/hf-space/src/components/megaeval/ShiftTab.tsx deleted file mode 100644 index 0e97fd4155cfd993d7a89d3cd0e82218ac05d003..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/ShiftTab.tsx +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Distribution shift panel: cohorts whose share of the evaluation population - * diverges from the training mix, annotated with the recall gap so a shift - * that also hurts quality stands out. - */ -import Box from '@mui/material/Box'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import { fmtCompact, getRunShift } from '../../services/megaeval'; -import { usePoll } from '../../services/labeleval'; -import { ErrorNote, LoadingBox, SectionCard, fmtPct } from '../labeleval/shared'; - -function gapColor(gap: number | null): string | undefined { - if (gap === null) return undefined; - const g = Math.abs(gap); - if (g >= 0.1) return '#ef5350'; - if (g >= 0.05) return '#ffa726'; - return undefined; -} - -export default function ShiftTab({ runId, refreshKey }: { runId: string; refreshKey: number }) { - const shift = usePoll(() => getRunShift(runId), null, [runId, refreshKey]); - const data = shift.data; - - return ( - - {shift.error ? : null} - {shift.loading && !data ? : null} - {data ? ( - <> - - {data.method} · thresholds: min eval count {fmtCompact(data.thresholds.min_eval_count)}, relative change ≥{' '} - {fmtPct(data.thresholds.rel_threshold)} - - {data.shifts.length === 0 ? ( - - No significant distribution shifts detected. - - ) : ( - - - - Cohort - Train → eval share - Relative change - Eval count - Cohort recall vs overall - Recall gap - - - - {data.shifts.map((s) => { - const color = gapColor(s.recall_gap); - return ( - - {s.cohort} - - {fmtPct(s.train_share)} → {fmtPct(s.eval_share)} - - 0 ? '#ffa726' : '#4fc3f7' }} - > - {s.relative_change > 0 ? '+' : ''} - {Math.round(s.relative_change * 100)}% - - - {fmtCompact(s.eval_count)} - - - {fmtPct(s.cohort_recall)} vs {fmtPct(s.overall_recall)} - - - {s.recall_gap === null ? '—' : `${(s.recall_gap * 100).toFixed(1)} pts`} - - - ); - })} - -
    - )} - - - Highlighted rows are cohorts that are over-represented vs training and underperforming on - recall — prime candidates for targeted data collection. - - - - ) : null} -     - ); -} diff --git a/hf-space/src/components/megaeval/shared.tsx b/hf-space/src/components/megaeval/shared.tsx deleted file mode 100644 index d33825fa827b915e43ab720a9864bf3111f172b0..0000000000000000000000000000000000000000 --- a/hf-space/src/components/megaeval/shared.tsx +++ /dev/null @@ -1,304 +0,0 @@ -/** - * Shared UI helpers for the Evaluation Command Center (megaeval). - * Complements components/labeleval/shared.tsx with aggregate-first idioms: - * query provenance badges, exact/approx honesty tags, delta coloring, - * outcome chips and sketch histogram sparklines. - */ -import type { ReactNode } from 'react'; -import Box from '@mui/material/Box'; -import Chip from '@mui/material/Chip'; -import Tooltip from '@mui/material/Tooltip'; -import Typography from '@mui/material/Typography'; -import { CheckCircle2, AlertTriangle, OctagonAlert } from 'lucide-react'; -import type { ErrorType, HistogramSketch, QueryMeta, QueryRow } from '../../types/megaeval'; -import { fmtCompact } from '../../services/megaeval'; -import { fmtPct } from '../labeleval/shared'; - -// ---------------------------------------------------------------- query provenance - -const SOURCE_COLORS: Record = { - cache: { bg: '#1b5e20', fg: '#a5d6a7' }, - cube: { bg: '#0d47a1', fg: '#90caf9' }, - scan: { bg: '#e65100', fg: '#ffe0b2' }, -}; - -function fmtLatency(ms: number): string { - if (ms < 1) return `${ms.toFixed(1)}ms`; - if (ms < 100) return `${ms.toFixed(1)}ms`; - return `${Math.round(ms)}ms`; -} - -/** "cube · 74ms" / "cache · 0.1ms" provenance badge for POST /api/evaluations/query panels. */ -export function QueryBadge({ meta }: { meta: QueryMeta | null | undefined }) { - if (!meta) return null; - const colors = SOURCE_COLORS[meta.source] ?? SOURCE_COLORS.cube; - const label = `${meta.source} · ${fmtLatency(meta.latency_ms)}`; - return ( - - - - ); -} - -/** Exact vs sketch-derived honesty tag. */ -export function ExactnessTag({ approx, sx }: { approx: boolean; sx?: object }) { - return ( - - ); -} - -// ---------------------------------------------------------------- query row helpers - -/** Reads a numeric metric out of a QueryRow (values arrive as string | number | null). */ -export function rowNum(row: QueryRow, key: string): number | null { - const v = row[key]; - if (v === null || v === undefined) return null; - const n = typeof v === 'number' ? v : Number(v); - return Number.isNaN(n) ? null : n; -} - -export function rowStr(row: QueryRow, key: string): string { - const v = row[key]; - return v === null || v === undefined ? '—' : String(v); -} - -// ---------------------------------------------------------------- inline percent bar - -/** Compact percentage cell with an inline bar, for dense aggregate tables. */ -export function PctBarCell({ - value, - color = '#4fc3f7', - width = 84, -}: { - value: number | null | undefined; - color?: string; - width?: number; -}) { - if (value === null || value === undefined || Number.isNaN(value)) { - return ( - - — - - ); - } - const frac = Math.max(0, Math.min(1, value <= 1 ? value : value / 100)); - return ( - - - - - - {fmtPct(value)} - - - ); -} - -// ---------------------------------------------------------------- outcome / error type chips - -export const OUTCOME_COLORS: Record = { - TP: { bg: '#1b5e20', fg: '#a5d6a7' }, - FN: { bg: '#b71c1c', fg: '#ffcdd2' }, - FP: { bg: '#e65100', fg: '#ffe0b2' }, - LOCALIZATION: { bg: '#4a148c', fg: '#ce93d8' }, - ANOMALY: { bg: '#880e4f', fg: '#f48fb1' }, - LOW_CONF: { bg: '#f9a825', fg: '#212121' }, -}; - -export function OutcomeChip({ outcome }: { outcome: string }) { - const colors = OUTCOME_COLORS[outcome] ?? { bg: '#37474f', fg: '#cfd8dc' }; - return ( - - ); -} - -export const ERROR_TYPE_COLORS: Record = { - FN: '#ef5350', - FP: '#ffa726', - LOCALIZATION: '#ab47bc', - ANOMALY: '#ec407a', - LOW_CONF: '#ffee58', -}; - -// ---------------------------------------------------------------- run status - -const RUN_STATUS_COLORS: Record = { - created: { bg: '#37474f', fg: '#cfd8dc' }, - queued: { bg: '#37474f', fg: '#cfd8dc' }, - running: { bg: '#0d47a1', fg: '#90caf9' }, - reducing: { bg: '#4a148c', fg: '#ce93d8' }, - materializing: { bg: '#004d40', fg: '#80cbc4' }, - published: { bg: '#1b5e20', fg: '#a5d6a7' }, - failed: { bg: '#b71c1c', fg: '#ffcdd2' }, -}; - -export function RunStatusChip({ status }: { status: string }) { - const colors = RUN_STATUS_COLORS[status] ?? { bg: '#37474f', fg: '#cfd8dc' }; - return ( - - ); -} - -// ---------------------------------------------------------------- container status - -export function ContainerStatusIcon({ status }: { status: 'ok' | 'warn' | 'critical' }) { - if (status === 'critical') return ; - if (status === 'warn') return ; - return ; -} - -// ---------------------------------------------------------------- deltas - -/** Colored delta text. Set higherIsBetter=false for metrics like anomaly_rate / error_rate / fp. */ -export function DeltaText({ - delta, - higherIsBetter = true, - asPct = true, -}: { - delta: number; - higherIsBetter?: boolean; - asPct?: boolean; -}) { - const good = higherIsBetter ? delta >= 0 : delta <= 0; - const neutral = Math.abs(delta) < 1e-9; - const magnitude = asPct ? (Math.abs(delta) * 100).toFixed(1) : Math.abs(delta).toFixed(3); - const sign = delta > 0 ? '+' : delta < 0 ? '−' : '±'; - const text = `${sign}${magnitude}`; - return ( - - {text} - - ); -} - -// ---------------------------------------------------------------- dimension chips - -export function DimChips({ values, sx }: { values: (string | undefined)[]; sx?: object }) { - return ( - - {values - .filter((v): v is string => Boolean(v)) - .map((v, i) => ( - - ))} - - ); -} - -// ---------------------------------------------------------------- histogram sparkline (sketch) - -/** SVG histogram sparkline with p10 / p50 / p90 markers. Sketch-derived — always labeled approx. */ -export function HistogramSparkline({ - sketch, - color = '#4fc3f7', - width = 260, - height = 56, -}: { - sketch: HistogramSketch; - color?: string; - width?: number; - height?: number; -}) { - const maxCount = Math.max(1, ...sketch.counts); - const barW = width / Math.max(1, sketch.counts.length); - const span = sketch.hi - sketch.lo || 1; - const xFor = (v: number) => ((v - sketch.lo) / span) * width; - const markers: Array<{ key: string; v: number }> = ['p10', 'p50', 'p90'] - .filter((k) => sketch.percentiles[k] !== undefined) - .map((k) => ({ key: k, v: sketch.percentiles[k] })); - return ( - - - {sketch.counts.map((c, i) => { - const h = (c / maxCount) * height; - return ( - - ); - })} - {markers.map((m) => ( - - - - {m.key} {m.v.toFixed(2)} - - - ))} - - - ); -} - -// ---------------------------------------------------------------- misc - -export function CompactStat({ label, value }: { label: string; value: ReactNode }) { - return ( - - - {label} - - - {value} - - - ); -} - -export function compactWithExact(n: number | null | undefined): string { - if (n === null || n === undefined) return '—'; - return `${fmtCompact(n)} (${Math.round(n).toLocaleString()})`; -} diff --git a/hf-space/src/components/platform/ContainerQualityPanel.tsx b/hf-space/src/components/platform/ContainerQualityPanel.tsx deleted file mode 100644 index c19ded0db15d8c03060bf84b5bf315661f4b52a6..0000000000000000000000000000000000000000 --- a/hf-space/src/components/platform/ContainerQualityPanel.tsx +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Container quality cards/table with verification rates (Phase 1 foundation). - * Extends MegaEval ContainersTab data via /api/containers/quality. - */ -import Box from '@mui/material/Box'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import { useContainerQuality } from '../../services/platform'; -import { ErrorNote, LoadingBox, MetricCard, SectionCard, fmtPct } from '../labeleval/shared'; - -export default function ContainerQualityPanel({ - runId, - refreshKey = 0, -}: { - runId: string | null; - refreshKey?: number; -}) { - const poll = useContainerQuality(runId, refreshKey); - - if (!runId) { - return ( - - - Select a published evaluation run to load container quality profiles. - - - ); - } - if (poll.error && !poll.data) return ; - if (poll.loading && !poll.data) return ; - const data = poll.data; - if (!data) return null; - - const v = data.verification_headline || data.summary?.verification; - - return ( - - - {data.total} containers · {data.model_version} - - } - > - - - - - - - - - - - - - - - Container - Status - P - R - F1 - IoU - Verified - HITL - Risk - - - - {data.containers.slice(0, 20).map((c) => ( - - #{c.container_id} - {c.status ?? '—'} - {fmtPct(c.precision)} - {fmtPct(c.recall)} - {fmtPct(c.f1)} - {fmtPct(c.mean_iou)} - {fmtPct(c.verification?.verified_rate)} - {fmtPct(c.verification?.hitl_rate)} - {c.risk_score.toFixed(2)} - - ))} - -
    -     -     - ); -} diff --git a/hf-space/src/components/platform/GateStatusPanel.tsx b/hf-space/src/components/platform/GateStatusPanel.tsx deleted file mode 100644 index 721eac7b264020fbce10c0a10b1d213eb81ddf2d..0000000000000000000000000000000000000000 --- a/hf-space/src/components/platform/GateStatusPanel.tsx +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Multi-gate readiness skeleton: Scenario / Coverage / Regression / Safety / Release - * (+ quality / launch when wired). - */ -import Box from '@mui/material/Box'; -import Chip from '@mui/material/Chip'; -import Typography from '@mui/material/Typography'; -import { CheckCircle2, CircleDashed, XCircle } from 'lucide-react'; -import { useGateStatus } from '../../services/platform'; -import { ErrorNote, LoadingBox, SectionCard } from '../labeleval/shared'; - -const DISPLAY_ORDER = ['scenario', 'coverage', 'regression', 'safety', 'quality', 'launch', 'release']; - -function statusChip(passed: boolean | null, ready: boolean) { - if (!ready) { - return } label="Skeleton" variant="outlined" />; - } - if (passed === true) { - return } label="Pass" />; - } - if (passed === false) { - return } label="Fail" />; - } - return ; -} - -export default function GateStatusPanel({ - candidateRunId, - baselineRunId, - refreshKey = 0, -}: { - candidateRunId: string | null; - baselineRunId: string | null; - refreshKey?: number; -}) { - const poll = useGateStatus(candidateRunId, baselineRunId, refreshKey); - - if (poll.error && !poll.data) return ; - if (poll.loading && !poll.data) return ; - const data = poll.data; - if (!data) return null; - - const byType = Object.fromEntries(data.gates.map((g) => [g.gate_type, g])); - - return ( - - {data.summary.release_ready - ? 'Release ready' - : `${data.summary.unwired_count} gates still Phase 2–6 stubs`} - - } - > - - {DISPLAY_ORDER.map((type) => { - const g = byType[type]; - if (!g) return null; - return ( - - - {type} - - {statusChip(g.passed, g.ready)} - - {g.message} - - - ); - })} - - - ); -} diff --git a/hf-space/src/components/platform/ModelComparePanel.tsx b/hf-space/src/components/platform/ModelComparePanel.tsx deleted file mode 100644 index 49c85cd2b75606624c8aaa814b9b8624e5f2d76e..0000000000000000000000000000000000000000 --- a/hf-space/src/components/platform/ModelComparePanel.tsx +++ /dev/null @@ -1,117 +0,0 @@ -/** - * Multi-model compare panel (A vs B vs C) — wraps /api/models/compare. - */ -import { useMemo } from 'react'; -import Alert from '@mui/material/Alert'; -import Box from '@mui/material/Box'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import { compareModels } from '../../services/platform'; -import { usePoll } from '../../services/labeleval'; -import { ErrorNote, LoadingBox, SectionCard, fmtPct } from '../labeleval/shared'; - -export default function ModelComparePanel({ - runId, - baselineRunId, - extraRunIds = [], - refreshKey = 0, -}: { - runId: string | null; - baselineRunId: string | null; - extraRunIds?: string[]; - refreshKey?: number; -}) { - const runIds = useMemo(() => { - const ids = [baselineRunId, runId, ...extraRunIds].filter(Boolean) as string[]; - return [...new Set(ids)]; - }, [runId, baselineRunId, extraRunIds]); - - const cmp = usePoll( - () => - runIds.length >= 2 - ? compareModels(runIds, baselineRunId) - : Promise.resolve(null), - null, - [runIds.join(','), baselineRunId, refreshKey] - ); - - if (runIds.length < 2) { - return ( - - Select a baseline run (and optionally more candidates) to compare models A / B / C. - - ); - } - if (cmp.error && !cmp.data) return ; - if (cmp.loading && !cmp.data) return ; - const data = cmp.data; - if (!data) return null; - - const modelNames = data.models.map((m) => m.model_version); - - return ( - - - Baseline {data.baseline_run_id.slice(0, 8)}… - - } - > - - {data.recommendations.map((r) => ( - - {r.candidate}: {r.recommendation} - {r.blocker_count ? ` (${r.blocker_count} blockers)` : ''} - - ))} - - - - - - Metric - {modelNames.map((m) => ( - - {m} - - ))} - - - - {data.metric_matrix.map((row) => ( - - {row.metric} - {modelNames.map((m) => { - const v = row[m] as number | null | undefined; - const d = row.deltas_vs_baseline?.[m]; - return ( - - {fmtPct(v ?? null)} - {d !== undefined && d !== null ? ( - - {d >= 0 ? '+' : ''} - {(d * 100).toFixed(1)}pp - - ) : null} - - ); - })} - - ))} - -
    -     -     - ); -} diff --git a/hf-space/src/context/FilterContext.tsx b/hf-space/src/context/FilterContext.tsx deleted file mode 100644 index 339d48ea15c549ab36b9dd995488f68b27efca9c..0000000000000000000000000000000000000000 --- a/hf-space/src/context/FilterContext.tsx +++ /dev/null @@ -1,153 +0,0 @@ -import React, { - createContext, - useCallback, - useContext, - useEffect, - useMemo, - useRef, - useState, -} from 'react'; -import { fetchStatewide } from '../services/api'; -import type { SSAMRecord, StatewideResponse } from '../types'; - -export interface Filters { - counties: string[]; - conflictTypes: string[]; - severityLabels: string[]; - ttcMax: number | null; - speedMin: number | null; - search: string; -} - -const DEFAULT_FILTERS: Filters = { - counties: [], - conflictTypes: [], - severityLabels: [], - ttcMax: null, - speedMin: null, - search: '', -}; - -interface FilterContextValue { - filters: Filters; - setFilters: (patch: Partial) => void; - resetFilters: () => void; - page: number; - setPage: (page: number) => void; - pageSize: number; - setPageSize: (size: number) => void; - sortBy: string; - sortDir: 'asc' | 'desc'; - setSort: (column: string) => void; - data: StatewideResponse | null; - loading: boolean; - error: string | null; - selected: SSAMRecord | null; - setSelected: (record: SSAMRecord | null) => void; - refresh: () => void; -} - -const FilterContext = createContext(null); - -export function FilterProvider({ children }: { children: React.ReactNode }) { - const [filters, setFiltersState] = useState(DEFAULT_FILTERS); - const [page, setPage] = useState(1); - const [pageSize, setPageSize] = useState(25); - const [sortBy, setSortBy] = useState('severity_index'); - const [sortDir, setSortDir] = useState<'asc' | 'desc'>('desc'); - const [data, setData] = useState(null); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(null); - const [selected, setSelected] = useState(null); - const [refreshTick, setRefreshTick] = useState(0); - - const setFilters = useCallback((patch: Partial) => { - setFiltersState((prev) => ({ ...prev, ...patch })); - setPage(1); - }, []); - - const resetFilters = useCallback(() => { - setFiltersState(DEFAULT_FILTERS); - setPage(1); - }, []); - - const setSort = useCallback((column: string) => { - setSortBy((prevBy) => { - setSortDir((prevDir) => (prevBy === column && prevDir === 'desc' ? 'asc' : 'desc')); - return column; - }); - setPage(1); - }, []); - - const refresh = useCallback(() => setRefreshTick((t) => t + 1), []); - - const abortRef = useRef(null); - - useEffect(() => { - // Debounce so fast typing in the search box issues a single request. - const timer = setTimeout(() => { - abortRef.current?.abort(); - const controller = new AbortController(); - abortRef.current = controller; - setLoading(true); - fetchStatewide( - { - counties: filters.counties.length ? filters.counties : undefined, - conflict_types: filters.conflictTypes.length ? filters.conflictTypes : undefined, - severity_labels: filters.severityLabels.length ? filters.severityLabels : undefined, - ttc_max: filters.ttcMax, - speed_min: filters.speedMin, - search: filters.search || undefined, - page, - page_size: pageSize, - sort_by: sortBy, - sort_dir: sortDir, - }, - controller.signal, - ) - .then((res) => { - setData(res); - setError(null); - }) - .catch((err: unknown) => { - if ((err as Error).name !== 'AbortError') { - setError(err instanceof Error ? err.message : String(err)); - } - }) - .finally(() => { - if (abortRef.current === controller) setLoading(false); - }); - }, 250); - return () => clearTimeout(timer); - }, [filters, page, pageSize, sortBy, sortDir, refreshTick]); - - const value = useMemo( - () => ({ - filters, - setFilters, - resetFilters, - page, - setPage, - pageSize, - setPageSize, - sortBy, - sortDir, - setSort, - data, - loading, - error, - selected, - setSelected, - refresh, - }), - [filters, setFilters, resetFilters, page, pageSize, sortBy, sortDir, setSort, data, loading, error, selected, refresh], - ); - - return {children}; -} - -export function useFilters(): FilterContextValue { - const ctx = useContext(FilterContext); - if (!ctx) throw new Error('useFilters must be used within FilterProvider'); - return ctx; -} diff --git a/hf-space/src/context/LabelEvalContext.tsx b/hf-space/src/context/LabelEvalContext.tsx deleted file mode 100644 index f1eb89a640548f29aaf1139525ae6663f65b1451..0000000000000000000000000000000000000000 --- a/hf-space/src/context/LabelEvalContext.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import { createContext, useContext } from 'react'; -import type { StreamEvent } from '../types/labeleval'; - -export type PageId = - | 'command' - | 'overview' - | 'datasets' - | 'label-generation' - | 'rare-events' - | 'quality' - | 'regression' - | 'triage' - | 'review' - | 'training' - | 'models' - | 'evaluation' - | 'audit' - | 'pipeline' - | 'ssam' - | 'legacy'; - -export const ALL_PAGE_IDS: PageId[] = [ - 'command', - 'overview', - 'datasets', - 'label-generation', - 'rare-events', - 'quality', - 'regression', - 'triage', - 'review', - 'training', - 'models', - 'evaluation', - 'audit', - 'pipeline', - 'ssam', - 'legacy', -]; - -/** Maps backend evidence_link.page strings (unknown vocabulary) onto our page ids. */ -export function resolvePageId(page: string): PageId { - const normalized = page.toLowerCase().replace(/_/g, '-'); - if ((ALL_PAGE_IDS as string[]).includes(normalized)) return normalized as PageId; - const aliases: Record = { - 'rare-event': 'rare-events', - rareevents: 'rare-events', - 'human-review': 'review', - hitl: 'review', - 'review-tasks': 'review', - 'quality-engine': 'quality', - 'label-eval': 'evaluation', - evaluations: 'evaluation', - model: 'models', - dataset: 'datasets', - train: 'training', - 'pipeline-architecture': 'pipeline', - }; - return aliases[normalized] ?? 'overview'; -} - -export interface LabelEvalContextValue { - activeDatasetId: string | null; - setActiveDatasetId: (id: string | null) => void; - /** Latest event from the SSE stream (or the polling fallback). */ - stream: StreamEvent | null; - page: PageId; - /** Optional entity id carried by navigation (e.g. review task id). */ - entityId: string | null; - navigate: (page: PageId, entityId?: string | null) => void; -} - -export const LabelEvalContext = createContext(null); - -export function useLabelEval(): LabelEvalContextValue { - const ctx = useContext(LabelEvalContext); - if (!ctx) throw new Error('useLabelEval must be used within the LabelEval provider'); - return ctx; -} diff --git a/hf-space/src/index.tsx b/hf-space/src/index.tsx deleted file mode 100644 index 9500e54680bbbe18ca60e2d11644ea2a2ce3ac81..0000000000000000000000000000000000000000 --- a/hf-space/src/index.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import React from 'react'; -import { createRoot } from 'react-dom/client'; -import CssBaseline from '@mui/material/CssBaseline'; -import { ThemeProvider, createTheme } from '@mui/material/styles'; -import App from './App'; -import './styles/globals.css'; - -const theme = createTheme({ - palette: { - mode: 'dark', - background: { default: '#101418', paper: '#161b21' }, - primary: { main: '#4fc3f7' }, - }, - typography: { - fontFamily: "'Inter', 'Helvetica Neue', Arial, sans-serif", - }, -}); - -const container = document.getElementById('root'); -if (!container) throw new Error('Root element missing'); -const root = createRoot(container); -root.render( - - - - - - -); diff --git a/hf-space/src/pages/AuditPage.tsx b/hf-space/src/pages/AuditPage.tsx deleted file mode 100644 index 1f98208e6cf8a89244ad620ca5179b4ee2661c9f..0000000000000000000000000000000000000000 --- a/hf-space/src/pages/AuditPage.tsx +++ /dev/null @@ -1,109 +0,0 @@ -import Box from '@mui/material/Box'; -import Typography from '@mui/material/Typography'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import { getProcessUnits, getAudit, usePoll } from '../services/labeleval'; -import { - SectionCard, - MetricCard, - HBar, - LoadingBox, - ErrorNote, - fmtInt, - fmtNum, -} from '../components/labeleval/shared'; - -export default function AuditPage() { - const units = usePoll(getProcessUnits, 5000); - const audit = usePoll(() => getAudit(200), 5000); - - const stages = units.data ? Object.entries(units.data.by_stage).sort((a, b) => b[1] - a[1]) : []; - const maxStage = Math.max(1, ...stages.map(([, v]) => v)); - - return ( - - - {units.error && !units.data ? : null} - {units.data ? ( - <> - - - - - - - {stages.length > 0 ? ( - <> - - BY STAGE - - - {stages.map(([stage, v]) => ( - - ))} - - - ) : ( - - No per-stage consumption yet. - - )} - - ) : !units.error ? ( - - ) : null} - - - - {audit.error && !audit.data ? : null} - {audit.data && audit.data.events.length > 0 ? ( - - - - Timestamp - Actor - Action - Entity - Detail - - - - {audit.data.events.map((e) => ( - - - {new Date(e.timestamp).toLocaleString()} - - {e.actor} - {e.action} - - {e.entity_type}/{e.entity_id} - - {e.detail} - - ))} - -
    - ) : audit.data ? ( - - No audit events yet — every dataset generation, run, review action and training job is recorded here. - - ) : null} -     -     - ); -} diff --git a/hf-space/src/pages/CommandCenterPage.tsx b/hf-space/src/pages/CommandCenterPage.tsx deleted file mode 100644 index 18c7ff67598258158533d90d7c09f96cfc404999..0000000000000000000000000000000000000000 --- a/hf-space/src/pages/CommandCenterPage.tsx +++ /dev/null @@ -1,325 +0,0 @@ -/** - * Evaluation Command Center — the aggregate-first landing page. - * - * Mental model surfaced in the UI: - * Dataset (population) → Evaluation Run → Population → Cohort → Container → Annotation - * Aggregates come first; individual annotations are the deepest drill-down. - * - * Runs are async: launched runs are queued, progress streams over SSE - * (stream.megaeval.active_runs) with a 1.5s polling fallback on GET /runs/{id}, - * and the page auto-refreshes when the run publishes. - */ -import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import Alert from '@mui/material/Alert'; -import Box from '@mui/material/Box'; -import Tab from '@mui/material/Tab'; -import Tabs from '@mui/material/Tabs'; -import Typography from '@mui/material/Typography'; -import type { EvaluationRunInfo, ReviewState, RunStatus } from '../types/megaeval'; -import { - getCacheStats, - getPopulations, - getRun, - getRunReview, - getRuns, -} from '../services/megaeval'; -import { usePoll } from '../services/labeleval'; -import { useLabelEval } from '../context/LabelEvalContext'; -import { EmptyState, ErrorNote, LoadingBox, SectionCard } from '../components/labeleval/shared'; -import HeaderBar, { - GeneratePopulationButton, - NewRunButton, -} from '../components/megaeval/HeaderBar'; -import RunProgressCard from '../components/megaeval/RunProgressCard'; -import HeroRow from '../components/megaeval/HeroRow'; -import QualityTab from '../components/megaeval/QualityTab'; -import CohortTab from '../components/megaeval/CohortTab'; -import ContainersTab from '../components/megaeval/ContainersTab'; -import InvestigationTab from '../components/megaeval/InvestigationTab'; -import CompareTab from '../components/megaeval/CompareTab'; -import ReviewTab from '../components/megaeval/ReviewTab'; -import ShiftTab from '../components/megaeval/ShiftTab'; -import LineageCard from '../components/megaeval/LineageCard'; - -const ACTIVE_STATUSES: RunStatus[] = ['created', 'queued', 'running', 'reducing', 'materializing']; - -type TabId = - | 'quality' - | 'cohorts' - | 'containers' - | 'investigation' - | 'compare' - | 'review' - | 'shift' - | 'lineage'; - -export default function CommandCenterPage() { - const { stream, entityId, navigate } = useLabelEval(); - - const populationsPoll = usePoll(getPopulations, 15000); - const runsPoll = usePoll(getRuns, 8000); - const cachePoll = usePoll(getCacheStats, 10000); - - const populations = useMemo(() => { - const list = populationsPoll.data?.populations ?? []; - return [...list].sort((a, b) => (b.created_at ?? '').localeCompare(a.created_at ?? '')); - }, [populationsPoll.data]); - const runs = useMemo(() => runsPoll.data?.runs ?? [], [runsPoll.data]); - - const [populationId, setPopulationId] = useState(null); - const [runId, setRunId] = useState(entityId); - const [baselineRunId, setBaselineRunId] = useState(null); - const [tab, setTab] = useState('quality'); - const [drillContainerId, setDrillContainerId] = useState(null); - const [publishTick, setPublishTick] = useState(0); - const [review, setReview] = useState(null); - const [selectedRun, setSelectedRun] = useState(null); - const [runError, setRunError] = useState(null); - - // Explicit selection persists the run in the URL hash (#/command/). - const selectRun = useCallback( - (id: string) => { - setRunId(id); - setDrillContainerId(null); - navigate('command', id); - }, - [navigate] - ); - - // Follow hash changes (back/forward navigation, deep links). - useEffect(() => { - if (entityId && entityId !== runId) { - setRunId(entityId); - setDrillContainerId(null); - } - }, [entityId]); // eslint-disable-line react-hooks/exhaustive-deps - - // Default selections once data arrives. - useEffect(() => { - if (!populationId && populations.length) setPopulationId(populations[0].population_id); - }, [populations, populationId]); - - useEffect(() => { - if (!runId && runs.length) { - const byCreated = [...runs].sort((a, b) => (b.created_at ?? '').localeCompare(a.created_at ?? '')); - const pick = byCreated.find((r) => r.status === 'published') ?? byCreated[0]; - if (pick) setRunId(pick.run_id); - } - }, [runs, runId]); - - // Selected run detail: 1.5s polling while the run is in flight (SSE-independent fallback). - useEffect(() => { - setSelectedRun(null); - setRunError(null); - if (!runId) return; - let cancelled = false; - let timer: ReturnType | null = null; - const load = async () => { - try { - const info = await getRun(runId); - if (cancelled) return; - setSelectedRun(info); - setRunError(null); - if (info.status !== 'published' && info.status !== 'failed') timer = setTimeout(() => void load(), 1500); - } catch (err) { - if (cancelled) return; - setRunError(err instanceof Error ? err.message : String(err)); - timer = setTimeout(() => void load(), 3000); - } - }; - void load(); - return () => { - cancelled = true; - if (timer) clearTimeout(timer); - }; - }, [runId]); - - // Auto-refresh all panels when the selected run publishes. - const prevStatusRef = useRef(null); - const runsRefresh = runsPoll.refresh; - useEffect(() => { - const status = selectedRun?.status ?? null; - if (status === 'published' && prevStatusRef.current && prevStatusRef.current !== 'published') { - setPublishTick((t) => t + 1); - runsRefresh(); - } - prevStatusRef.current = status; - }, [selectedRun?.status, runsRefresh]); - - const published = selectedRun?.status === 'published'; - - // Review state (shared by the hero CI annotations and the Review tab). - useEffect(() => { - setReview(null); - if (!runId || !published) return; - let cancelled = false; - getRunReview(runId) - .then((r) => { - if (!cancelled) setReview(r); - }) - .catch(() => undefined); - return () => { - cancelled = true; - }; - }, [runId, published, publishTick]); - - const megaStatus = stream?.megaeval ?? null; - const cache = megaStatus?.cache ?? cachePoll.data ?? null; - - // Prefer SSE progress for the selected run when it is fresher than the poll. - const streamProgress = megaStatus?.active_runs.find((r) => r.run_id === runId) ?? null; - const selectedInFlight = - selectedRun && ACTIVE_STATUSES.includes(selectedRun.status) ? streamProgress ?? selectedRun : null; - const otherActiveRuns = (megaStatus?.active_runs ?? []).filter((r) => r.run_id !== runId); - - const openContainer = useCallback((cid: number) => { - setDrillContainerId(cid); - setTab('containers'); - }, []); - - const initialLoading = - populationsPoll.loading && !populationsPoll.data && runsPoll.loading && !runsPoll.data; - const noPopulation = populationsPoll.data !== null && populations.length === 0; - const noRuns = !noPopulation && runsPoll.data !== null && runs.length === 0; - - const onPopulationGenerated = useCallback( - (pop: { population_id: string }) => { - populationsPoll.refresh(); - setPopulationId(pop.population_id); - }, - [populationsPoll] - ); - - const onRunCreated = useCallback( - (run: EvaluationRunInfo) => { - runsRefresh(); - selectRun(run.run_id); - }, - [runsRefresh, selectRun] - ); - - if (initialLoading) return ; - - return ( - - {populationsPoll.error && !populationsPoll.data ? : null} - - - - - Dataset (population) → Evaluation run → Population → Cohort → Container → Annotation — aggregates first, - individual annotations only at the deepest drill-down. - - - {/* Empty states: nothing generated / no runs yet */} - {noPopulation ? ( - - - - - } - /> - - ) : null} - - {noRuns ? ( - - - - -     - } - /> - - ) : null} - - {runError && !selectedRun ? : null} - - {/* Live progress for the selected run + any other active runs */} - {selectedInFlight ? : null} - {otherActiveRuns.map((r) => ( - - ))} - - {selectedRun?.status === 'failed' ? ( - - Run {selectedRun.run_id.slice(0, 8)} failed{selectedRun.error ? `: ${selectedRun.error}` : '.'} Launch a new - evaluation run from the header. - - ) : null} - - {/* Hero aggregates + tab sections (published runs only) */} - {published && selectedRun ? ( - <> - - - setTab(v)} - variant="scrollable" - scrollButtons="auto" - sx={{ - minHeight: 36, - borderBottom: '1px solid #232a31', - '& .MuiTab-root': { minHeight: 36, py: 0.5 }, - }} - > - - - - - - - - - - - {tab === 'quality' ? ( - - ) : null} - {tab === 'cohorts' ? : null} - {tab === 'containers' ? ( - - ) : null} - {tab === 'investigation' ? ( - - ) : null} - {tab === 'compare' ? ( - - ) : null} - {tab === 'review' ? ( - - ) : null} - {tab === 'shift' ? : null} - {tab === 'lineage' ? : null} - - ) : null} - - ); -} diff --git a/hf-space/src/pages/DatasetsPage.tsx b/hf-space/src/pages/DatasetsPage.tsx deleted file mode 100644 index 7e79840bdc8eaa15f818e06f6e7cdd0462cd4e1a..0000000000000000000000000000000000000000 --- a/hf-space/src/pages/DatasetsPage.tsx +++ /dev/null @@ -1,201 +0,0 @@ -import { useEffect, useState } from 'react'; -import Box from '@mui/material/Box'; -import Card from '@mui/material/Card'; -import CardActionArea from '@mui/material/CardActionArea'; -import CardContent from '@mui/material/CardContent'; -import Typography from '@mui/material/Typography'; -import type { QualityGroupDetail } from '../types/labeleval'; -import { - getDatasets, - getQualityGroups, - getQualityGroupDetail, - usePoll, -} from '../services/labeleval'; -import { useLabelEval } from '../context/LabelEvalContext'; -import DatasetSelector from '../components/labeleval/DatasetSelector'; -import { - SectionCard, - LoadingBox, - ErrorNote, - HBar, - fmtInt, - fmtPct, - fmtNum, - MetricCard, -} from '../components/labeleval/shared'; - -const GROUP_COLORS: Record = { - verified: '#66bb6a', - non_verified: '#ffa726', - hitl: '#42a5f5', - rejected: '#ef5350', -}; - -export default function DatasetsPage() { - const { activeDatasetId, setActiveDatasetId } = useLabelEval(); - const datasets = usePoll(getDatasets, 5000); - const groups = usePoll( - () => getQualityGroups(activeDatasetId), - activeDatasetId ? 5000 : null, - [activeDatasetId] - ); - const [groupDetail, setGroupDetail] = useState(null); - const [detailLoading, setDetailLoading] = useState(false); - const [detailError, setDetailError] = useState(null); - const [verificationRates, setVerificationRates] = useState>({}); - - // DatasetSummary carries no verification rate, so pull it from each - // dataset's quality groups whenever the dataset list changes. - const datasetIdsKey = (datasets.data?.datasets ?? []).map((d) => d.dataset_id).join(','); - useEffect(() => { - let cancelled = false; - const ids = datasetIdsKey ? datasetIdsKey.split(',') : []; - if (ids.length === 0) return; - Promise.all( - ids.map((id) => - getQualityGroups(id) - .then((g) => [id, g.verification_rate] as const) - .catch(() => null) - ) - ).then((entries) => { - if (cancelled) return; - const next: Record = {}; - for (const e of entries) { - if (e) next[e[0]] = e[1]; - } - setVerificationRates(next); - }); - return () => { - cancelled = true; - }; - }, [datasetIdsKey]); - - useEffect(() => { - setGroupDetail(null); - setDetailError(null); - }, [activeDatasetId]); - - const openGroup = async (groupId: string) => { - setDetailLoading(true); - setDetailError(null); - try { - setGroupDetail(await getQualityGroupDetail(groupId)); - } catch (err) { - setDetailError(err instanceof Error ? err.message : String(err)); - } finally { - setDetailLoading(false); - } - }; - - const failureRows = groupDetail ? Object.entries(groupDetail.failure_reason_counts).sort((a, b) => b[1] - a[1]) : []; - const maxFailure = Math.max(1, ...failureRows.map(([, c]) => c)); - - return ( - - {datasets.loading && !datasets.data ? : null} - {datasets.error && !datasets.data ? : null} - - {datasets.data ? ( - - ) : null} - - {activeDatasetId ? ( - - {groups.error && !groups.data ? : null} - {groups.data ? ( - <> - - {fmtInt(groups.data.total)} annotations · verification rate {fmtPct(groups.data.verification_rate)}. - Click a group for its quality profile. - - - {groups.data.groups.map((g) => ( - - void openGroup(g.group_id)}> - - - {g.name.replace(/_/g, ' ')} - - - {fmtInt(g.count)} - - - {fmtPct(g.pct)} of annotations - - - - - ))} - - - ) : !groups.error ? ( - - ) : null} - - {detailLoading ? : null} - {detailError ? : null} - {groupDetail && !detailLoading ? ( - - - {groupDetail.name.replace(/_/g, ' ').toUpperCase()} — quality profile - - - - - - - - - - - {failureRows.length > 0 ? ( - <> - - FAILURE REASON BREAKDOWN - - - {failureRows.map(([reason, count]) => ( - - ))} - - - ) : ( - - No failure reasons recorded for this group. - - )} - - ) : null} - - ) : ( - - Select a dataset above to drill into its quality groups. - - )} - - ); -} diff --git a/hf-space/src/pages/EvaluationPage.tsx b/hf-space/src/pages/EvaluationPage.tsx deleted file mode 100644 index 34c13e6ca562e86c2129b78575a502d001754e8e..0000000000000000000000000000000000000000 --- a/hf-space/src/pages/EvaluationPage.tsx +++ /dev/null @@ -1,198 +0,0 @@ -import { useEffect, useState } from 'react'; -import Box from '@mui/material/Box'; -import Select from '@mui/material/Select'; -import MenuItem from '@mui/material/MenuItem'; -import Typography from '@mui/material/Typography'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import TablePagination from '@mui/material/TablePagination'; -import type { CopilotExplainRequest, EvaluationRecord } from '../types/labeleval'; -import { - getDatasets, - getQualityGroups, - getQualityGroupDetail, - getEvaluation, - usePoll, -} from '../services/labeleval'; -import { useLabelEval } from '../context/LabelEvalContext'; -import EvidencePanel from '../components/labeleval/EvidencePanel'; -import CopilotDrawer from '../components/labeleval/CopilotDrawer'; -import { SectionCard, StatusChip, LoadingBox, ErrorNote, fmtNum, fmtPct } from '../components/labeleval/shared'; - -const PAGE_SIZE = 15; - -export default function EvaluationPage() { - const { activeDatasetId, setActiveDatasetId } = useLabelEval(); - const datasets = usePoll(getDatasets, 10000); - - const [annotationIds, setAnnotationIds] = useState([]); - const [idsLoading, setIdsLoading] = useState(false); - const [idsError, setIdsError] = useState(null); - const [page, setPage] = useState(0); - const [records, setRecords] = useState([]); - const [recordsLoading, setRecordsLoading] = useState(false); - - const [selected, setSelected] = useState(null); - const [panelOpen, setPanelOpen] = useState(false); - const [copilotOpen, setCopilotOpen] = useState(false); - const [copilotRequest, setCopilotRequest] = useState(null); - - // Collect annotation ids for the chosen dataset via its quality groups. - useEffect(() => { - let cancelled = false; - setAnnotationIds([]); - setPage(0); - setIdsError(null); - if (!activeDatasetId) return; - setIdsLoading(true); - (async () => { - try { - const groups = await getQualityGroups(activeDatasetId); - const details = await Promise.all( - groups.groups.filter((g) => g.count > 0).map((g) => getQualityGroupDetail(g.group_id)) - ); - const ids = details.flatMap((d) => d.annotation_ids); - if (!cancelled) setAnnotationIds(ids); - } catch (err) { - if (!cancelled) setIdsError(err instanceof Error ? err.message : String(err)); - } finally { - if (!cancelled) setIdsLoading(false); - } - })(); - return () => { - cancelled = true; - }; - }, [activeDatasetId]); - - // Load the current page of evaluation records. - useEffect(() => { - let cancelled = false; - const pageIds = annotationIds.slice(page * PAGE_SIZE, (page + 1) * PAGE_SIZE); - if (pageIds.length === 0) { - setRecords([]); - return; - } - setRecordsLoading(true); - Promise.all(pageIds.map((id) => getEvaluation(id).catch(() => null))) - .then((rows) => { - if (!cancelled) setRecords(rows.filter((r): r is EvaluationRecord => r !== null)); - }) - .finally(() => { - if (!cancelled) setRecordsLoading(false); - }); - return () => { - cancelled = true; - }; - }, [annotationIds, page]); - - return ( - - - - - Dataset: - - - - - {idsLoading ? : null} - {idsError ? : null} - - {!activeDatasetId ? ( - - Pick a dataset to explore its annotation-level evaluation records. - - ) : annotationIds.length === 0 && !idsLoading ? ( - - No evaluation records for this dataset yet — run the evaluation pipeline first. - - ) : ( - <> - {recordsLoading ? : null} - - - - Annotation - Class - Model - 3D IoU - Anomaly - Consensus - Status - Primary failure - - - - {records.map((r) => ( - { - setSelected(r); - setPanelOpen(true); - }} - > - {r.annotation_id} - {r.object_class} - {r.model_version} - - {fmtNum(r.geometry.iou_3d)} - - - {fmtNum(r.anomaly.score)} - - - {fmtPct(r.grading.consensus)} - - {r.decision ? : '—'} - - {r.decision?.primary_failure_reason?.replace(/_/g, ' ') ?? '—'} - - - ))} - -
    - setPage(p)} - rowsPerPage={PAGE_SIZE} - rowsPerPageOptions={[PAGE_SIZE]} - /> - - )} -     - - setPanelOpen(false)} - record={selected} - onAskCopilot={(request) => { - setCopilotRequest(request); - setCopilotOpen(true); - }} - /> - setCopilotOpen(false)} request={copilotRequest} /> -     - ); -} diff --git a/hf-space/src/pages/HumanReviewPage.tsx b/hf-space/src/pages/HumanReviewPage.tsx deleted file mode 100644 index 31fec39e5eda69ae5f8ebc4b86c19164db13a49f..0000000000000000000000000000000000000000 --- a/hf-space/src/pages/HumanReviewPage.tsx +++ /dev/null @@ -1,86 +0,0 @@ -import { useEffect, useState } from 'react'; -import Box from '@mui/material/Box'; -import type { FrameSummary, ReviewTask } from '../types/labeleval'; -import { getReviewTasks, getReviewTask, getFrame, usePoll } from '../services/labeleval'; -import { useLabelEval } from '../context/LabelEvalContext'; -import HITLReview from '../components/labeleval/HITLReview'; -import RelabelingPanel from '../components/labeleval/RelabelingPanel'; -import { LoadingBox, ErrorNote } from '../components/labeleval/shared'; - -export default function HumanReviewPage() { - const { entityId } = useLabelEval(); - const tasks = usePoll(getReviewTasks, 5000); - const [selectedTask, setSelectedTask] = useState(null); - const [frame, setFrame] = useState(null); - const [prevFrame, setPrevFrame] = useState(null); - const [nextFrame, setNextFrame] = useState(null); - const [frameLoading, setFrameLoading] = useState(false); - const [loadError, setLoadError] = useState(null); - - const selectTask = async (taskId: string) => { - setLoadError(null); - setFrameLoading(true); - setFrame(null); - setPrevFrame(null); - setNextFrame(null); - try { - const task = await getReviewTask(taskId); - setSelectedTask(task); - const res = await getFrame(task.frame_id); - setFrame(res.frame); - const [prev, next] = await Promise.all([ - res.prev ? getFrame(res.prev).then((r) => r.frame).catch(() => null) : Promise.resolve(null), - res.next ? getFrame(res.next).then((r) => r.frame).catch(() => null) : Promise.resolve(null), - ]); - setPrevFrame(prev); - setNextFrame(next); - } catch (err) { - setLoadError(err instanceof Error ? err.message : String(err)); - } finally { - setFrameLoading(false); - } - }; - - // Deep-link support: navigate('review', taskId). - useEffect(() => { - if (entityId) void selectTask(entityId); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [entityId]); - - const annotation = - (frame && selectedTask - ? frame.annotations.find((a) => a.annotation_id === selectedTask.annotation_id) - : null) ?? null; - - return ( - - {tasks.loading && !tasks.data ? : null} - {tasks.error && !tasks.data ? : null} - {loadError ? : null} - - void selectTask(id)} - frame={frame} - prevFrame={prevFrame} - nextFrame={nextFrame} - frameLoading={frameLoading} - /> - - { - tasks.refresh(); - if (selectedTask) { - // Reload the task so its resolution/status is up to date. - getReviewTask(selectedTask.task_id) - .then(setSelectedTask) - .catch(() => undefined); - } - }} - /> - - ); -} diff --git a/hf-space/src/pages/LabelGenerationPage.tsx b/hf-space/src/pages/LabelGenerationPage.tsx deleted file mode 100644 index 079726f499f7d2abf99c795fbf943c80a40fcf55..0000000000000000000000000000000000000000 --- a/hf-space/src/pages/LabelGenerationPage.tsx +++ /dev/null @@ -1,171 +0,0 @@ -import { useState } from 'react'; -import Box from '@mui/material/Box'; -import List from '@mui/material/List'; -import ListItemButton from '@mui/material/ListItemButton'; -import ListItemText from '@mui/material/ListItemText'; -import Select from '@mui/material/Select'; -import MenuItem from '@mui/material/MenuItem'; -import Typography from '@mui/material/Typography'; -import Alert from '@mui/material/Alert'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import type { FrameResponse } from '../services/labeleval'; -import { getDatasets, getFrameIds, getFrame, usePoll } from '../services/labeleval'; -import { useLabelEval } from '../context/LabelEvalContext'; -import InputStage from '../components/labeleval/InputStage'; -import { SectionCard, StatusChip, LoadingBox, ErrorNote, fmtNum } from '../components/labeleval/shared'; - -export default function LabelGenerationPage() { - const { activeDatasetId, setActiveDatasetId } = useLabelEval(); - const datasets = usePoll(getDatasets, 10000); - const frameIds = usePoll( - () => (activeDatasetId ? getFrameIds(activeDatasetId) : Promise.resolve({ frame_ids: [] as string[] })), - null, - [activeDatasetId] - ); - const [selectedFrame, setSelectedFrame] = useState(null); - const [frameData, setFrameData] = useState(null); - const [frameLoading, setFrameLoading] = useState(false); - const [frameError, setFrameError] = useState(null); - - const activeDataset = datasets.data?.datasets.find((d) => d.dataset_id === activeDatasetId) ?? null; - - const loadFrame = async (frameId: string) => { - setSelectedFrame(frameId); - setFrameLoading(true); - setFrameError(null); - try { - setFrameData(await getFrame(frameId)); - } catch (err) { - setFrameData(null); - setFrameError(err instanceof Error ? err.message : String(err)); - } finally { - setFrameLoading(false); - } - }; - - return ( - - - - - - Auto-generated labels are hypotheses, not truth — every annotation below must pass the - evaluation pipeline (anomaly detection, grading, strict validation and triage) before it can be verified. - - - - - Dataset: - - - - - {!activeDatasetId ? ( - - Select a dataset to browse its frames. - - ) : ( - - - - FRAMES ({frameIds.data?.frame_ids.length ?? 0}) - - {frameIds.loading ? : null} - {frameIds.error && !frameIds.data ? : null} - - {(frameIds.data?.frame_ids ?? []).map((fid) => ( - void loadFrame(fid)} - > - - - ))} - - - - - {frameLoading ? : null} - {frameError ? : null} - {frameData && !frameLoading ? ( - <> - - {frameData.frame.frame_id} — scene {frameData.frame.scene_id}, seq {frameData.frame.sequence_id} - - - t = {frameData.frame.timestamp_us.toLocaleString()} µs ·{' '} - {frameData.frame.num_lidar_points.toLocaleString()} lidar points · ego speed{' '} - {fmtNum(frameData.frame.ego_pose.speed_mps, 1)} m/s · prev: {frameData.prev ?? '—'} · next:{' '} - {frameData.next ?? '—'} - - - - - Annotation - Class - Confidence - Model - Version - Status - - - - {frameData.frame.annotations.map((a) => ( - - {a.annotation_id} - {a.class_name} - - {fmtNum(a.confidence)} - - {a.model} - {a.model_version} - - - - - ))} - -
    - {frameData.frame.annotations.length === 0 ? ( - - No annotations in this frame. - - ) : null} - - ) : !frameLoading && !frameError ? ( - - Select a frame to inspect its auto-label hypotheses. - - ) : null} -     -     - )} -     -     - ); -} diff --git a/hf-space/src/pages/LegacyStudioPage.tsx b/hf-space/src/pages/LegacyStudioPage.tsx deleted file mode 100644 index 32ea89d541479d65bf7a723baea14e13070b3bc5..0000000000000000000000000000000000000000 --- a/hf-space/src/pages/LegacyStudioPage.tsx +++ /dev/null @@ -1,44 +0,0 @@ -import Box from '@mui/material/Box'; -import Button from '@mui/material/Button'; -import Typography from '@mui/material/Typography'; -import { ExternalLink } from 'lucide-react'; - -// In dev the vite server only proxies /api, so '/' would render the React app -// itself; in production the backend serves the legacy UI at the site root. -const LEGACY_URL = import.meta.env.DEV ? 'http://localhost:8000/' : '/'; - -export default function LegacyStudioPage() { - return ( - - - - Legacy vanilla-JS studio served by the backend at {LEGACY_URL}. If the frame below is blank, make sure the - backend is running on port 8000. - - - -