Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /Daylily-Informatics_bloom /bloom_lims /observability.py
| from __future__ import annotations | |
| import os | |
| import socket | |
| from collections import Counter, deque | |
| from dataclasses import dataclass, field | |
| from datetime import UTC, datetime | |
| from math import ceil | |
| from threading import RLock | |
| from typing import Any | |
| from fastapi import Request | |
| from bloom_lims import __version__ | |
| from bloom_lims.api.v1.dependencies import APIUser | |
| from bloom_lims.config import get_settings | |
| from bloom_lims.schema_drift import read_schema_drift_report | |
| from bloom_lims.tapdb_dag import bloom_tapdb_dag_obs_services_fragment | |
| CONTRACT_VERSION = "v3" | |
| SERVICE_NAME = "bloom" | |
| def _utcnow() -> str: | |
| return datetime.now(UTC).isoformat() | |
| def _instance_id() -> str: | |
| return f"{socket.gethostname()}:{os.getpid()}" | |
| def _build_sha() -> str: | |
| return ( | |
| os.environ.get("BLOOM_BUILD_SHA") | |
| or os.environ.get("BUILD_SHA") | |
| or os.environ.get("GIT_SHA") | |
| or "" | |
| ) | |
| def _percentile(values: list[float], quantile: float) -> float: | |
| if not values: | |
| return 0.0 | |
| ordered = sorted(values) | |
| index = max(0, min(len(ordered) - 1, ceil(len(ordered) * quantile) - 1)) | |
| return round(float(ordered[index]), 3) | |
| class ProjectionMetadata: | |
| state: str = "ready" | |
| stale: bool = False | |
| observed_at: str | None = None | |
| last_synced_at: str | None = None | |
| detail: str | None = None | |
| def model_dump(self) -> dict[str, Any]: | |
| return { | |
| "state": self.state, | |
| "stale": self.stale, | |
| "observed_at": self.observed_at, | |
| "last_synced_at": self.last_synced_at, | |
| "detail": self.detail, | |
| } | |
| class EndpointRollup: | |
| method: str | |
| route_template: str | |
| request_count: int = 0 | |
| error_count: int = 0 | |
| status_class_counts: Counter[str] = field(default_factory=Counter) | |
| durations_ms: deque[float] = field(default_factory=lambda: deque(maxlen=512)) | |
| fingerprints: set[str] = field(default_factory=set) | |
| observed_at: str = field(default_factory=_utcnow) | |
| def record(self, *, status_code: int, duration_ms: float, fingerprint: str) -> None: | |
| self.request_count += 1 | |
| if status_code >= 500: | |
| self.error_count += 1 | |
| self.status_class_counts[f"{status_code // 100}xx"] += 1 | |
| self.durations_ms.append(float(duration_ms)) | |
| if fingerprint: | |
| self.fingerprints.add(fingerprint) | |
| self.observed_at = _utcnow() | |
| def to_dict(self) -> dict[str, Any]: | |
| durations = list(self.durations_ms) | |
| return { | |
| "method": self.method, | |
| "route_template": self.route_template, | |
| "request_count": self.request_count, | |
| "error_count": self.error_count, | |
| "status_class_counts": dict(self.status_class_counts), | |
| "p50_ms": _percentile(durations, 0.50), | |
| "p95_ms": _percentile(durations, 0.95), | |
| "p99_ms": _percentile(durations, 0.99), | |
| "fingerprint_count": len(self.fingerprints), | |
| "observed_at": self.observed_at, | |
| } | |
| class FamilyRollup: | |
| family: str | |
| request_count: int = 0 | |
| error_count: int = 0 | |
| durations_ms: deque[float] = field(default_factory=lambda: deque(maxlen=512)) | |
| observed_at: str = field(default_factory=_utcnow) | |
| def record(self, *, status_code: int, duration_ms: float) -> None: | |
| self.request_count += 1 | |
| if status_code >= 500: | |
| self.error_count += 1 | |
| self.durations_ms.append(float(duration_ms)) | |
| self.observed_at = _utcnow() | |
| def to_dict(self) -> dict[str, Any]: | |
| durations = list(self.durations_ms) | |
| return { | |
| "family": self.family, | |
| "request_count": self.request_count, | |
| "error_count": self.error_count, | |
| "p50_ms": _percentile(durations, 0.50), | |
| "p95_ms": _percentile(durations, 0.95), | |
| "p99_ms": _percentile(durations, 0.99), | |
| "observed_at": self.observed_at, | |
| } | |
| class BloomObservabilityStore: | |
| def __init__(self) -> None: | |
| self._lock = RLock() | |
| self._started_at = _utcnow() | |
| self._endpoint_rollups: dict[tuple[str, str], EndpointRollup] = {} | |
| self._family_rollups: dict[str, FamilyRollup] = {} | |
| self._db_probes: deque[dict[str, Any]] = deque(maxlen=25) | |
| self._auth_recent: deque[dict[str, Any]] = deque(maxlen=25) | |
| self._auth_status_counts: Counter[str] = Counter() | |
| self._obs_services_snapshot = self._build_obs_services_snapshot() | |
| def started_at(self) -> str: | |
| return self._started_at | |
| def _configured_service_dependencies(self) -> list[str]: | |
| settings = get_settings() | |
| configured: list[str] = [] | |
| if str(settings.atlas.base_url or "").strip(): | |
| configured.append("atlas") | |
| if bool(settings.dewey.enabled) and str(settings.dewey.base_url or "").strip(): | |
| configured.append("dewey") | |
| if str(settings.zebra_day.base_url or "").strip(): | |
| configured.append("zebra_day") | |
| return configured | |
| def _managed_services(self) -> list[dict[str, Any]]: | |
| return [] | |
| def _build_obs_services_snapshot(self) -> dict[str, Any]: | |
| snapshot = { | |
| "status": "ok", | |
| "endpoints": [ | |
| {"path": "/healthz", "auth": "none", "kind": "liveness"}, | |
| {"path": "/readyz", "auth": "none", "kind": "readiness"}, | |
| { | |
| "path": "/health", | |
| "auth": "operator_or_service_token", | |
| "kind": "summary", | |
| }, | |
| { | |
| "path": "/obs_services", | |
| "auth": "operator_or_service_token", | |
| "kind": "discovery", | |
| }, | |
| { | |
| "path": "/api_health", | |
| "auth": "operator_or_service_token", | |
| "kind": "api_rollup", | |
| }, | |
| { | |
| "path": "/endpoint_health", | |
| "auth": "operator_or_service_token", | |
| "kind": "endpoint_rollup", | |
| }, | |
| { | |
| "path": "/db_health", | |
| "auth": "operator_or_service_token", | |
| "kind": "database", | |
| }, | |
| { | |
| "path": "/api/anomalies", | |
| "auth": "operator_or_service_token", | |
| "kind": "anomaly_list", | |
| }, | |
| { | |
| "path": "/api/anomalies/{anomaly_id}", | |
| "auth": "operator_or_service_token", | |
| "kind": "anomaly_detail", | |
| }, | |
| {"path": "/my_health", "auth": "authenticated_self", "kind": "self"}, | |
| { | |
| "path": "/auth_health", | |
| "auth": "operator_or_service_token", | |
| "kind": "auth", | |
| }, | |
| ], | |
| "extensions": [ | |
| "bloom.admin_metrics_ui", | |
| "bloom.admin_observability_ui", | |
| "bloom.anomalies_v1", | |
| ], | |
| "capabilities": [], | |
| "external_ref_models": [], | |
| "dependencies": { | |
| "configured_services": self._configured_service_dependencies(), | |
| "observed_services": [], | |
| }, | |
| "managed_services": self._managed_services(), | |
| "observed_at": self._started_at, | |
| } | |
| fragment = bloom_tapdb_dag_obs_services_fragment() | |
| known_endpoints = { | |
| (str(item.get("path") or ""), str(item.get("kind") or "")) | |
| for item in snapshot["endpoints"] | |
| } | |
| known_extensions = {str(item) for item in snapshot["extensions"]} | |
| known_capabilities: set[str] = set() | |
| known_external_ref_models: set[str] = set() | |
| for item in list(fragment.get("endpoints") or []): | |
| if not isinstance(item, dict): | |
| continue | |
| key = (str(item.get("path") or ""), str(item.get("kind") or "")) | |
| if key in known_endpoints: | |
| continue | |
| snapshot["endpoints"].append(dict(item)) | |
| known_endpoints.add(key) | |
| for item in list(fragment.get("extensions") or []): | |
| normalized = str(item or "").strip() | |
| if not normalized or normalized in known_extensions: | |
| continue | |
| snapshot["extensions"].append(normalized) | |
| known_extensions.add(normalized) | |
| for item in list(fragment.get("capabilities") or []): | |
| normalized = str(item or "").strip() | |
| if not normalized or normalized in known_capabilities: | |
| continue | |
| snapshot["capabilities"].append(normalized) | |
| known_capabilities.add(normalized) | |
| for item in list(fragment.get("external_ref_models") or []): | |
| normalized = str(item or "").strip() | |
| if not normalized or normalized in known_external_ref_models: | |
| continue | |
| snapshot["external_ref_models"].append(normalized) | |
| known_external_ref_models.add(normalized) | |
| contract_version = str(fragment.get("contract_version") or "").strip() | |
| if contract_version: | |
| snapshot["tapdb_dag_contract_version"] = contract_version | |
| return snapshot | |
| def projection( | |
| self, *, observed_at: str | None = None, detail: str | None = None | |
| ) -> ProjectionMetadata: | |
| seen_at = observed_at or self._started_at | |
| return ProjectionMetadata( | |
| state="ready", | |
| stale=False, | |
| observed_at=seen_at, | |
| last_synced_at=seen_at, | |
| detail=detail, | |
| ) | |
| def record_http_request( | |
| self, | |
| *, | |
| method: str, | |
| route_template: str, | |
| status_code: int, | |
| duration_ms: float, | |
| ) -> None: | |
| family = self._classify_family(route_template) | |
| key = (method.upper(), route_template) | |
| fingerprint = f"{method.upper()}:{route_template}:{status_code // 100}xx" | |
| with self._lock: | |
| endpoint_rollup = self._endpoint_rollups.setdefault( | |
| key, | |
| EndpointRollup(method=method.upper(), route_template=route_template), | |
| ) | |
| endpoint_rollup.record( | |
| status_code=status_code, | |
| duration_ms=duration_ms, | |
| fingerprint=fingerprint, | |
| ) | |
| family_rollup = self._family_rollups.setdefault( | |
| family, FamilyRollup(family=family) | |
| ) | |
| family_rollup.record(status_code=status_code, duration_ms=duration_ms) | |
| def record_db_probe(self, *, status: str, latency_ms: float, detail: str) -> None: | |
| with self._lock: | |
| self._db_probes.appendleft( | |
| { | |
| "status": status, | |
| "latency_ms": round(float(latency_ms), 3), | |
| "detail": str(detail or ""), | |
| "observed_at": _utcnow(), | |
| } | |
| ) | |
| def record_auth_event( | |
| self, | |
| *, | |
| status: str, | |
| mode: str, | |
| detail: str, | |
| service_principal: bool, | |
| principal_email: str | None = None, | |
| ) -> None: | |
| event = { | |
| "status": status, | |
| "mode": mode, | |
| "detail": str(detail or ""), | |
| "service_principal": service_principal, | |
| "principal_email": str(principal_email or ""), | |
| "observed_at": _utcnow(), | |
| } | |
| with self._lock: | |
| self._auth_recent.appendleft(event) | |
| self._auth_status_counts[status] += 1 | |
| def obs_services_snapshot(self) -> tuple[ProjectionMetadata, dict[str, Any]]: | |
| snapshot = dict(self._obs_services_snapshot) | |
| dependencies = snapshot.get("dependencies") | |
| if isinstance(dependencies, dict): | |
| snapshot["dependencies"] = { | |
| "configured_services": list( | |
| dependencies.get("configured_services") or [] | |
| ), | |
| "observed_services": list(dependencies.get("observed_services") or []), | |
| } | |
| snapshot["managed_services"] = [ | |
| dict(item) for item in snapshot.get("managed_services") or [] | |
| ] | |
| observed_at = str(snapshot.get("observed_at") or self._started_at) | |
| return self.projection(observed_at=observed_at), snapshot | |
| def api_health(self) -> tuple[ProjectionMetadata, list[dict[str, Any]]]: | |
| with self._lock: | |
| families = [rollup.to_dict() for rollup in self._family_rollups.values()] | |
| families.sort(key=lambda item: (-int(item["request_count"]), item["family"])) | |
| observed_at = families[0]["observed_at"] if families else self._started_at | |
| return self.projection(observed_at=observed_at), families | |
| def endpoint_health( | |
| self, *, offset: int, limit: int | |
| ) -> tuple[ProjectionMetadata, dict[str, Any]]: | |
| with self._lock: | |
| items = [rollup.to_dict() for rollup in self._endpoint_rollups.values()] | |
| items.sort( | |
| key=lambda item: ( | |
| -int(item["request_count"]), | |
| item["route_template"], | |
| item["method"], | |
| ) | |
| ) | |
| total = len(items) | |
| sliced = items[offset : offset + limit] | |
| observed_at = ( | |
| sliced[0]["observed_at"] | |
| if sliced | |
| else (items[0]["observed_at"] if items else self._started_at) | |
| ) | |
| return self.projection(observed_at=observed_at), { | |
| "total": total, | |
| "offset": offset, | |
| "limit": limit, | |
| "items": sliced, | |
| } | |
| def latest_db_probe(self) -> dict[str, Any] | None: | |
| with self._lock: | |
| return dict(self._db_probes[0]) if self._db_probes else None | |
| def db_health(self) -> tuple[ProjectionMetadata, dict[str, Any]]: | |
| from bloom_lims.tapdb_metrics import build_metrics_page_context | |
| target_label = "target" | |
| metrics_ctx = build_metrics_page_context(target_label, limit=1000) | |
| summary = dict(metrics_ctx.get("summary") or {}) | |
| latest = self.latest_db_probe() | |
| schema_drift = read_schema_drift_report(target=target_label) | |
| observed_at = ( | |
| (latest or {}).get("observed_at") | |
| or str(summary.get("last_seen") or "") | |
| or str(schema_drift.get("checked_at") or "") | |
| or self._started_at | |
| ) | |
| payload = { | |
| "status": str((latest or {}).get("status") or "unknown"), | |
| "latest": latest, | |
| "recent": list(summary.get("slowest") or [])[:25], | |
| "slowest": list(summary.get("slowest") or [])[:10], | |
| "hottest": list(summary.get("by_path") or [])[:10], | |
| "by_path": list(summary.get("by_path") or [])[:25], | |
| "by_table": list(summary.get("by_table") or [])[:25], | |
| "metrics_enabled": bool(metrics_ctx.get("metrics_enabled", False)), | |
| "metrics_message": str(metrics_ctx.get("metrics_message") or ""), | |
| "schema_drift": schema_drift, | |
| "observed_at": observed_at, | |
| } | |
| return self.projection(observed_at=observed_at), payload | |
| def auth_health(self) -> tuple[ProjectionMetadata, dict[str, Any]]: | |
| settings = get_settings() | |
| with self._lock: | |
| recent = list(self._auth_recent) | |
| status_counts = dict(self._auth_status_counts) | |
| latest = recent[0] if recent else None | |
| observed_at = str((latest or {}).get("observed_at") or self._started_at) | |
| recent_user_count = len( | |
| { | |
| str(item.get("principal_email") or "").strip().lower() | |
| for item in recent | |
| if str(item.get("principal_email") or "").strip() | |
| and not bool(item.get("service_principal")) | |
| } | |
| ) | |
| return self.projection(observed_at=observed_at), { | |
| "status": str((latest or {}).get("status") or "unknown"), | |
| "mode": str((latest or {}).get("mode") or "unknown"), | |
| "cognito_configured": bool( | |
| settings.auth.cognito_domain | |
| and settings.auth.cognito_user_pool_id | |
| and settings.auth.cognito_client_id | |
| ), | |
| "cognito_domain": str(settings.auth.cognito_domain or ""), | |
| "user_pool_id": str(settings.auth.cognito_user_pool_id or ""), | |
| "app_client_id_present": bool(settings.auth.cognito_client_id), | |
| "recent": recent, | |
| "status_counts": status_counts, | |
| "sessions": { | |
| "supported": True, | |
| "active_session_count": None, | |
| "recent_user_count": recent_user_count, | |
| "observed_at": observed_at, | |
| }, | |
| "observed_at": observed_at, | |
| } | |
| def _classify_family(self, route_template: str) -> str: | |
| path = route_template or "/" | |
| if path.startswith("/api/v1/"): | |
| parts = [part for part in path.split("/") if part] | |
| return parts[2] if len(parts) > 2 else "api" | |
| if path.startswith("/auth"): | |
| return "auth" | |
| if path.startswith("/admin"): | |
| return "admin" | |
| if path in { | |
| "/health", | |
| "/healthz", | |
| "/readyz", | |
| "/obs_services", | |
| "/api_health", | |
| "/endpoint_health", | |
| "/db_health", | |
| "/my_health", | |
| "/auth_health", | |
| }: | |
| return "observability" | |
| return "web" | |
| def base_frame(request: Request, *, status: str) -> dict[str, Any]: | |
| settings = get_settings() | |
| return { | |
| "contract_version": CONTRACT_VERSION, | |
| "service": SERVICE_NAME, | |
| "environment": settings.environment, | |
| "instance_id": _instance_id(), | |
| "observed_at": _utcnow(), | |
| "status": status, | |
| "request_id": getattr(request.state, "request_id", ""), | |
| "correlation_id": getattr(request.state, "correlation_id", ""), | |
| "build": { | |
| "version": __version__, | |
| "sha": _build_sha(), | |
| }, | |
| } | |
| def _status_for_projection(projection: ProjectionMetadata, ready_status: str) -> str: | |
| return ready_status if projection.state == "ready" else "unknown" | |
| def _with_projection( | |
| payload: dict[str, Any], projection: ProjectionMetadata | |
| ) -> dict[str, Any]: | |
| payload["projection"] = projection.model_dump() | |
| return payload | |
| def _probe_projection(observed_at: str) -> ProjectionMetadata: | |
| return ProjectionMetadata( | |
| state="ready", | |
| stale=False, | |
| observed_at=observed_at, | |
| last_synced_at=observed_at, | |
| detail=None, | |
| ) | |
| def build_healthz_payload( | |
| request: Request, | |
| *, | |
| started_at: str, | |
| ) -> dict[str, Any]: | |
| payload = base_frame(request, status="ok") | |
| observed_at = str(payload.get("observed_at") or _utcnow()) | |
| payload["checks"] = { | |
| "process": { | |
| "status": "ok", | |
| "started_at": started_at, | |
| } | |
| } | |
| return _with_projection(payload, _probe_projection(observed_at)) | |
| def build_readyz_payload( | |
| request: Request, | |
| *, | |
| started_at: str, | |
| database_check: dict[str, Any], | |
| ready: bool, | |
| process_details: dict[str, Any] | None = None, | |
| ) -> dict[str, Any]: | |
| payload = base_frame(request, status="ok" if ready else "degraded") | |
| observed_at = str(payload.get("observed_at") or _utcnow()) | |
| payload["ready"] = ready | |
| payload["checks"] = { | |
| "process": { | |
| "status": "ok", | |
| "started_at": started_at, | |
| "details": dict(process_details or {}), | |
| }, | |
| "database": { | |
| "status": str(database_check.get("status") or "unknown"), | |
| "latency_ms": database_check.get("latency_ms"), | |
| "detail": database_check.get("detail"), | |
| "observed_at": database_check.get("observed_at") or observed_at, | |
| "details": dict(database_check.get("details") or {}), | |
| }, | |
| } | |
| return _with_projection(payload, _probe_projection(observed_at)) | |
| def build_health_payload( | |
| request: Request, *, projection: ProjectionMetadata, health_snapshot: dict[str, Any] | |
| ) -> dict[str, Any]: | |
| payload = base_frame( | |
| request, | |
| status=_status_for_projection( | |
| projection, str(health_snapshot.get("status") or "unknown") | |
| ), | |
| ) | |
| payload["checks"] = dict(health_snapshot.get("checks") or {}) | |
| return _with_projection(payload, projection) | |
| def build_obs_services_payload( | |
| request: Request, *, projection: ProjectionMetadata, snapshot: dict[str, Any] | |
| ) -> dict[str, Any]: | |
| payload = base_frame( | |
| request, | |
| status=_status_for_projection(projection, str(snapshot.get("status") or "ok")), | |
| ) | |
| payload["endpoints"] = list(snapshot.get("endpoints") or []) | |
| payload["extensions"] = list(snapshot.get("extensions") or []) | |
| payload["dependencies"] = { | |
| "configured_services": list( | |
| (snapshot.get("dependencies") or {}).get("configured_services") or [] | |
| ), | |
| "observed_services": list( | |
| (snapshot.get("dependencies") or {}).get("observed_services") or [] | |
| ), | |
| } | |
| payload["managed_services"] = list(snapshot.get("managed_services") or []) | |
| if snapshot.get("capabilities"): | |
| payload["capabilities"] = list(snapshot.get("capabilities") or []) | |
| if snapshot.get("external_ref_models"): | |
| payload["external_ref_models"] = list(snapshot.get("external_ref_models") or []) | |
| if snapshot.get("tapdb_dag_contract_version"): | |
| payload["tapdb_dag_contract_version"] = str( | |
| snapshot.get("tapdb_dag_contract_version") or "" | |
| ) | |
| return _with_projection(payload, projection) | |
| def build_api_health_payload( | |
| request: Request, *, projection: ProjectionMetadata, families: list[dict[str, Any]] | |
| ) -> dict[str, Any]: | |
| payload = base_frame(request, status=_status_for_projection(projection, "ok")) | |
| payload["families"] = families | |
| return _with_projection(payload, projection) | |
| def build_endpoint_health_payload( | |
| request: Request, | |
| *, | |
| projection: ProjectionMetadata, | |
| total: int, | |
| offset: int, | |
| limit: int, | |
| items: list[dict[str, Any]], | |
| ) -> dict[str, Any]: | |
| payload = base_frame(request, status=_status_for_projection(projection, "ok")) | |
| payload["page"] = {"total": total, "offset": offset, "limit": limit} | |
| payload["items"] = items | |
| return _with_projection(payload, projection) | |
| def build_db_health_payload( | |
| request: Request, *, projection: ProjectionMetadata, db_health: dict[str, Any] | |
| ) -> dict[str, Any]: | |
| payload = base_frame( | |
| request, | |
| status=_status_for_projection( | |
| projection, str(db_health.get("status") or "unknown") | |
| ), | |
| ) | |
| payload["database"] = db_health | |
| return _with_projection(payload, projection) | |
| def build_auth_health_payload( | |
| request: Request, *, projection: ProjectionMetadata, auth_rollup: dict[str, Any] | |
| ) -> dict[str, Any]: | |
| payload = base_frame( | |
| request, | |
| status=_status_for_projection( | |
| projection, str(auth_rollup.get("status") or "unknown") | |
| ), | |
| ) | |
| payload["auth"] = { | |
| "mode": str(auth_rollup.get("mode") or ""), | |
| "cognito_configured": bool(auth_rollup.get("cognito_configured", False)), | |
| "cognito_domain": str(auth_rollup.get("cognito_domain") or ""), | |
| "user_pool_id": str(auth_rollup.get("user_pool_id") or ""), | |
| "app_client_id_present": bool(auth_rollup.get("app_client_id_present", False)), | |
| "recent": list(auth_rollup.get("recent") or []), | |
| "status_counts": dict(auth_rollup.get("status_counts") or {}), | |
| "sessions": dict(auth_rollup.get("sessions") or {}), | |
| } | |
| return _with_projection(payload, projection) | |
| def _is_service_principal(user: APIUser) -> bool: | |
| return user.auth_source in {"token"} | |
| def build_my_health_payload(request: Request, user: APIUser) -> dict[str, Any]: | |
| payload = base_frame(request, status="ok") | |
| payload["principal"] = { | |
| "subject": str(user.user_id), | |
| "email": user.email, | |
| "name": None, | |
| "roles": user.roles, | |
| "auth_mode": user.auth_source, | |
| "expires_at": None, | |
| "service_principal": _is_service_principal(user), | |
| } | |
| return payload | |
Xet Storage Details
- Size:
- 24.4 kB
- Xet hash:
- ff1a0c92a0345431653454507455d399214e693f6bdecd054a9cf4b5cc9d1ba3
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.