diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..81527da5d46ca021461c65829f64ce3eadb3c356 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,27 @@ +.git +.gitignore + +.venv +__pycache__ +*.py[cod] +*.egg-info +.ruff_cache +.ipynb_checkpoints +.pytest_cache +.mypy_cache + +frontend/node_modules +frontend/dist +frontend/.vite + +data/raw +mlruns +notebooks + +.vscode +.idea +.DS_Store + +Dockerfile* +docker-compose*.yml +.dockerignore diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..966129f158b770b3f9eaec270969128cffdcfa69 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,10 @@ +data/processed/*.npy filter=lfs diff=lfs merge=lfs -text +data/processed/*.npz filter=lfs diff=lfs merge=lfs -text +models/*.pt filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text +*.webp filter=lfs diff=lfs merge=lfs -text +*.mp3 filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.wav filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..010c3b4a044cfbcaa8e7330252014ab5f90908d6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +.venv/ +__pycache__/ +*.py[cod] +.ipynb_checkpoints/ +.ruff_cache/ +*.egg-info/ +/data/raw +/mlruns/ +.DS_Store +.vscode/ +.idea/ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000000000000000000000000000000000000..2c0733315e415bfb5e5b353f9996ecd964d395b2 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..9efeac53976e1d68cd38e659995d50abb1ca03b6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,77 @@ +FROM node:20-alpine AS frontend + +RUN corepack enable && corepack prepare pnpm@9.12.0 --activate + +WORKDIR /web + +COPY frontend/package.json frontend/pnpm-lock.yaml ./ +RUN pnpm install --frozen-lockfile + +COPY frontend/ ./ +RUN pnpm build + + +FROM python:3.11-slim AS backend-builder + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + UV_LINK_MODE=copy \ + UV_PROJECT_ENVIRONMENT=/opt/venv + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:0.5.11 /uv /usr/local/bin/uv + +WORKDIR /app + +COPY pyproject.toml uv.lock ./ +COPY src ./src + +RUN uv sync --frozen --no-dev + + +FROM python:3.11-slim AS runtime + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH="/opt/venv/bin:$PATH" \ + HOME=/home/user \ + SADAR_PROCESSED_DIR=/app/data/processed \ + SADAR_MODELS_DIR=/app/models \ + SADAR_REPORTS_DIR=/app/reports \ + SADAR_SERVE_CONFIG=/app/configs/serve.yaml \ + SADAR_FRONTEND_DIR=/app/frontend/dist \ + PORT=7860 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgomp1 \ + curl \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --create-home --uid 1000 --shell /bin/bash user + +WORKDIR /app + +COPY --from=backend-builder /opt/venv /opt/venv +COPY --from=backend-builder /app/src /app/src +COPY --from=frontend /web/dist /app/frontend/dist + +COPY configs ./configs +COPY pyproject.toml ./ +COPY models ./models +COPY reports ./reports +COPY data/processed/scaler.npz data/processed/test.npy data/processed/val.npy ./data/processed/ + +RUN chown -R user:user /app + +USER user + +EXPOSE 7860 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ + CMD curl -fsS http://localhost:${PORT}/api/health || exit 1 + +CMD ["sh", "-c", "uvicorn sadar.serve.app:app --host 0.0.0.0 --port ${PORT}"] diff --git a/Dockerfile.backend b/Dockerfile.backend new file mode 100644 index 0000000000000000000000000000000000000000..41eec3df7d7b03dcef50f327143297ce19b5f519 --- /dev/null +++ b/Dockerfile.backend @@ -0,0 +1,59 @@ +FROM python:3.11-slim AS builder + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + UV_LINK_MODE=copy \ + UV_PROJECT_ENVIRONMENT=/opt/venv + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=ghcr.io/astral-sh/uv:0.5.11 /uv /usr/local/bin/uv + +WORKDIR /app + +COPY pyproject.toml uv.lock ./ +COPY src ./src + +RUN uv sync --frozen --no-dev + + +FROM python:3.11-slim AS runtime + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH="/opt/venv/bin:$PATH" \ + SADAR_PROCESSED_DIR=/app/data/processed \ + SADAR_MODELS_DIR=/app/models \ + SADAR_REPORTS_DIR=/app/reports \ + SADAR_SERVE_CONFIG=/app/configs/serve.yaml + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgomp1 \ + curl \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 1000 sadar \ + && useradd --system --uid 1000 --gid sadar --create-home sadar + +WORKDIR /app + +COPY --from=builder /opt/venv /opt/venv +COPY --from=builder /app/src /app/src + +COPY configs ./configs +COPY pyproject.toml ./ + +RUN mkdir -p /app/data/processed /app/models /app/reports \ + && chown -R sadar:sadar /app + +USER sadar + +EXPOSE 8000 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ + CMD curl -fsS http://localhost:8000/api/health || exit 1 + +CMD ["uvicorn", "sadar.serve.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f8f5c6585a237d0660ff5ab8aae89c1cc0b7d10c --- /dev/null +++ b/Makefile @@ -0,0 +1,85 @@ +.DEFAULT_GOAL := help +.PHONY: help install notebook preprocess baseline train-lstm train-transformer train-vae tune-vae evaluate compare serve web dev artifacts mlflow-ui lint clean docker-build docker-up docker-down docker-logs + +PROCESSED_DIR ?= data/processed +MODELS_DIR ?= models +VAE_CHECKPOINT ?= $(MODELS_DIR)/vae_lstm.pt +SCALER ?= $(PROCESSED_DIR)/scaler.npz + +help: ## Show this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-14s\033[0m %s\n", $$1, $$2}' + +install: ## Create the Python environment with uv + uv sync + +notebook: ## Open Jupyter for the notebooks (EDA, etc.) + uv run jupyter lab + +preprocess: ## Run the preprocessing pipeline (raw -> tensors) + uv run sadar-preprocess + +baseline: ## Fit and evaluate the Isolation Forest baseline + uv run sadar-baseline + +train-lstm: ## Train the LSTM autoencoder on the normal flights + uv run sadar-train-lstm + +train-transformer: ## Train the Transformer autoencoder on the normal flights + uv run sadar-train-transformer + +train-vae: ## Train the VAE-LSTM on the normal flights + uv run sadar-train-vae + +tune-vae: ## Search VAE-LSTM hyperparameters with Optuna + uv run sadar-tune-vae + +evaluate: ## Evaluate a trained autoencoder against synthetic anomalies + uv run sadar-evaluate + +compare: ## Compare every detector and select the final model + uv run sadar-compare + +artifacts: ## Ensure the preprocessed tensors and the VAE checkpoint exist + @if [ ! -f "$(SCALER)" ]; then \ + echo "[artifacts] $(SCALER) missing, running preprocess..."; \ + $(MAKE) preprocess; \ + fi + @if [ ! -f "$(VAE_CHECKPOINT)" ]; then \ + echo "[artifacts] $(VAE_CHECKPOINT) missing, trying to restore from MLflow..."; \ + uv run python -m sadar.models.restore || true; \ + fi + @if [ ! -f "$(VAE_CHECKPOINT)" ]; then \ + echo "[artifacts] still missing, training VAE-LSTM..."; \ + $(MAKE) train-vae; \ + fi + +serve: artifacts ## Run the inference API for the dashboard + uv run uvicorn sadar.serve.app:app --port 8000 + +web: ## Run the frontend dev server (Vite) + pnpm -C frontend dev + +dev: artifacts ## Run the backend API and the frontend dev server together + @trap 'kill 0' EXIT; uv run uvicorn sadar.serve.app:app --port 8000 & pnpm -C frontend dev + +mlflow-ui: ## Open the MLflow experiment tracking UI + uv run mlflow ui --backend-store-uri file:mlruns + +lint: ## Python linter + uv run ruff check src + +clean: ## Clean caches and artifacts + rm -rf .ruff_cache **/__pycache__ + +docker-build: ## Build the backend and frontend container images + docker compose build + +docker-up: ## Start the stack in the background (backend on :8000, frontend on :5180) + docker compose up -d + +docker-down: ## Stop the stack and remove the containers + docker compose down + +docker-logs: ## Tail the logs of the running stack + docker compose logs -f diff --git a/README.es.md b/README.es.md new file mode 100644 index 0000000000000000000000000000000000000000..c28c2703e484dbef93ae0a74168eb0998d467fc4 --- /dev/null +++ b/README.es.md @@ -0,0 +1,359 @@ +
+
+Smart Anomaly Detection for Aviation Routes
+ +Detección temprana de comportamiento de vuelo no conforme sobre Madrid-Barajas (LEMD).
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+ | MODEL | +REAL ROC | +REAL PR-AUC | +SYN ROC | +Δ vs BASELINE | +PR-AUC | +
|---|---|---|---|---|---|
|
+
+ {row.model}
+ {chosen ? " ★" : ""}
+
+
+ {MODEL_NOTES[row.model] ?? ""}
+
+ |
+ {pct(row.real_roc_auc)} | +{pct(row.real_pr_auc)} | +{pct(row.synthetic_mean_roc_auc)} | += baseline.real_pr_auc ? "var(--info)" : "var(--muted)", + }} + > + {baseline ? delta(row.real_pr_auc, baseline.real_pr_auc) : "—"} + | +
+
+
+
+ |
+
| ANOMALY | +INTENSITY | + {data.results.map((row) => ( ++ {row.model} + | + ))} +
|---|---|---|
| + {idx === 0 ? group.label : ""} + | +{row.intensity} | + {data.results.map((mdl) => { + const value = row.values[mdl.model]; + const isWinner = + Math.max(...Object.values(row.values)) === value && data.results.length > 1; + return ( ++ {pct(value)} + | + ); + })} +
{sub}
} +{body}
+
+ {subtitle}
+ +| \n", + " | dtype | \n", + "
|---|---|
| time | \n", + "int64 | \n", + "
| icao24 | \n", + "str | \n", + "
| lat | \n", + "float64 | \n", + "
| lon | \n", + "float64 | \n", + "
| baroaltitude | \n", + "float64 | \n", + "
| geoaltitude | \n", + "float64 | \n", + "
| velocity | \n", + "float64 | \n", + "
| heading | \n", + "float64 | \n", + "
| vertrate | \n", + "float64 | \n", + "
| callsign | \n", + "str | \n", + "
| onground | \n", + "bool | \n", + "
| squawk | \n", + "float64 | \n", + "
| alert | \n", + "bool | \n", + "
| spi | \n", + "bool | \n", + "
| lastcontact | \n", + "float64 | \n", + "
| flight_id | \n", + "str | \n", + "
| operation | \n", + "str | \n", + "
| time_utc | \n", + "datetime64[ms, UTC] | \n", + "
| velocity_kmh | \n", + "float64 | \n", + "
| dist_to_runway_m | \n", + "float64 | \n", + "
| flight_phase | \n", + "str | \n", + "
| source_date | \n", + "str | \n", + "
| \n", + " | min | \n", + "50% | \n", + "max | \n", + "
|---|---|---|---|
| baroaltitude | \n", + "-365.760000 | \n", + "4366.260000 | \n", + "38648.640000 | \n", + "
| geoaltitude | \n", + "-304.800000 | \n", + "4686.300000 | \n", + "39006.780000 | \n", + "
| velocity_kmh | \n", + "0.000000 | \n", + "527.218114 | \n", + "6153.974158 | \n", + "
| vertrate | \n", + "-139.476480 | \n", + "-0.650240 | \n", + "165.811200 | \n", + "
| dist_to_runway_m | \n", + "11.447052 | \n", + "35211.168852 | \n", + "199999.879356 | \n", + "
| heading | \n", + "-0.000000 | \n", + "197.007947 | \n", + "359.921405 | \n", + "
| \n", + " | null_% | \n", + "
|---|---|
| geoaltitude | \n", + "24.79 | \n", + "
| baroaltitude | \n", + "13.48 | \n", + "
| velocity | \n", + "13.37 | \n", + "
| heading | \n", + "13.37 | \n", + "
| velocity_kmh | \n", + "13.37 | \n", + "
| vertrate | \n", + "13.34 | \n", + "
| callsign | \n", + "2.45 | \n", + "
| squawk | \n", + "1.97 | \n", + "
| flight_id | \n", + "0.00 | \n", + "
| source_date | \n", + "0.00 | \n", + "
| flight_phase | \n", + "0.00 | \n", + "
| dist_to_runway_m | \n", + "0.00 | \n", + "
| time_utc | \n", + "0.00 | \n", + "
| operation | \n", + "0.00 | \n", + "
| time | \n", + "0.00 | \n", + "
| lastcontact | \n", + "0.00 | \n", + "
| spi | \n", + "0.00 | \n", + "
| alert | \n", + "0.00 | \n", + "
| icao24 | \n", + "0.00 | \n", + "
| onground | \n", + "0.00 | \n", + "
| lon | \n", + "0.00 | \n", + "
| lat | \n", + "0.00 | \n", + "
| dt | \n", + "0.00 | \n", + "
| \n", + " | rows | \n", + "
|---|---|
| flight_phase | \n", + "\n", + " |
| descent | \n", + "1090369 | \n", + "
| climb | \n", + "731531 | \n", + "
| on_ground | \n", + "684742 | \n", + "
| cruise | \n", + "446580 | \n", + "
| approach | \n", + "265192 | \n", + "
| takeoff | \n", + "211224 | \n", + "
| \n", + " | flight_id | \n", + "callsign | \n", + "date | \n", + "squawk(s) | \n", + "emergency_points | \n", + "total_points | \n", + "min_dist_to_rwy_km | \n", + "landed_on_ground | \n", + "
|---|---|---|---|---|---|---|---|---|
| 0 | \n", + "3420ca_1501530490 | \n", + "[IBE2845] | \n", + "20170731 | \n", + "[7500 (hijack)] | \n", + "3 | \n", + "171 | \n", + "0.8 | \n", + "False | \n", + "
| 1 | \n", + "4b161e_1580715790 | \n", + "[SWR202Y] | \n", + "20200203 | \n", + "[7700 (general emergency)] | \n", + "1 | \n", + "156 | \n", + "0.8 | \n", + "True | \n", + "
| 2 | \n", + "4ca564_1548700030 | \n", + "[RYR61AD, ] | \n", + "20190128 | \n", + "[7600 (radio failure)] | \n", + "1 | \n", + "173 | \n", + "0.3 | \n", + "True | \n", + "
| 3 | \n", + "738051_1506930700 | \n", + "[ELY395] | \n", + "20171002 | \n", + "[7700 (general emergency)] | \n", + "5 | \n", + "162 | \n", + "0.8 | \n", + "True | \n", + "