Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /ctrl-gaurav_effGen /effgen /server /app.py
| """effGen API Server — main FastAPI application with auth/RBAC/audit wired in. | |
| Usage:: | |
| # Production (requires EFFGEN_OIDC_ISSUER + EFFGEN_OIDC_CLIENT_ID) | |
| uvicorn effgen.server.app:create_app --factory --host 0.0.0.0 --port 8080 | |
| # Dev mode (auth disabled, loud warning) | |
| EFFGEN_DEV_MODE=1 uvicorn effgen.server.app:create_app --factory ... | |
| Environment variables | |
| --------------------- | |
| EFFGEN_DEV_MODE — "1" disables auth (warn only, never silently) | |
| EFFGEN_OIDC_ISSUER — OIDC issuer URL | |
| EFFGEN_OIDC_CLIENT_ID — JWT audience claim | |
| EFFGEN_OIDC_JWKS_URI — JWKS endpoint (auto-discovered if omitted) | |
| EFFGEN_RBAC_POLICY_FILE — path to JSON role policy file | |
| EFFGEN_AUDIT_DIR — override audit log directory | |
| EFFGEN_METRICS_AUTH — "1" to require auth on /metrics | |
| """ | |
| from __future__ import annotations | |
| import logging | |
| import os | |
| from typing import Any | |
| logger = logging.getLogger(__name__) | |
| # Machine-readable ``code`` for the errors the framework itself raises, so a | |
| # client can branch on a stable string instead of parsing the message. | |
| _HTTP_ERROR_CODES: dict[int, str] = { | |
| 404: "not_found", | |
| 405: "method_not_allowed", | |
| } | |
| # The application is assembled from sibling modules — middleware, the default | |
| # runner + model pool, the dashboard/playground surfaces, and the dashboard | |
| # data builders. Every name they define is re-exported here so | |
| # ``from effgen.server.app import X`` and patches against this module resolve | |
| # unchanged. | |
| from effgen.server.app_dashboard_data import ( # noqa: E402,F401 re-exported for import/patch parity | |
| _PROM_LINE_RE, | |
| _bucket_bounds, | |
| _build_dashboard_data, | |
| _build_dashboard_history, | |
| _collect_dashboard_metrics, | |
| _effgen_version, | |
| _get_prompt_templates, | |
| _get_recent_runs, | |
| _get_recent_spans, | |
| _histogram_quantile, | |
| _http_route_breakdown, | |
| _http_status_breakdown, | |
| _http_status_detail, | |
| _latency_percentiles, | |
| _metric_key, | |
| _model_breakdown, | |
| _parse_prometheus_labels, | |
| _parse_prometheus_text, | |
| _sum_samples, | |
| ) | |
| from effgen.server.app_middleware import ( # noqa: E402 | |
| MaxBodySizeMiddleware, | |
| RBACBudgetMiddleware, | |
| _enforce_max_body_size, # noqa: F401 re-exported for import/patch parity | |
| _reject_json, # noqa: F401 re-exported for import/patch parity | |
| ) | |
| from effgen.server.app_runner import ( # noqa: E402,F401 re-exported for import/patch parity | |
| _KNOWN_PROVIDERS, | |
| _LOADING_LOCKS, | |
| _MODEL_POOL, | |
| _MODEL_POOL_LOCK, | |
| _SERVED_MODEL_IDS, | |
| _SERVED_MODEL_LOCK, | |
| _SERVED_MODEL_MAX, | |
| _TRACE_ARG_MAX, | |
| _TRACE_RESULT_MAX, | |
| _build_default_runner, | |
| _extract_cost, | |
| _extract_tool_trace, | |
| _extract_usage, | |
| _get_pooled_model, | |
| _model_pool_max, | |
| _mount_existing_routers, | |
| _normalize_model_id, | |
| _record_served_model, | |
| _resolve_tools, | |
| _served_model_ids, | |
| _StreamWithUsage, | |
| _tool_name, | |
| ) | |
| from effgen.server.app_web import ( # noqa: E402 | |
| _PLAYGROUND_SAFE_TOOLS, # noqa: F401 re-exported for import/patch parity | |
| _asset_not_found, # noqa: F401 re-exported for import/patch parity | |
| _build_playground_bootstrap, # noqa: F401 re-exported for import/patch parity | |
| _FastAPIRequest, | |
| _mount_dashboard, | |
| _mount_playground, | |
| _resolve_web_asset, # noqa: F401 re-exported for import/patch parity | |
| ) | |
| def _server_version() -> str: | |
| """Return the effGen package version without paying for a heavy import. | |
| Reads the installed distribution metadata first (cheap; does not import the | |
| ``effgen`` package). Falls back to the package attribute, then a sentinel. | |
| """ | |
| try: | |
| from importlib.metadata import PackageNotFoundError, version | |
| try: | |
| return version("effgen") | |
| except PackageNotFoundError: # no install metadata (source/editable run); use the fallback version | |
| pass | |
| except Exception: # noqa: BLE001 - importlib.metadata always present on 3.11 | |
| pass | |
| try: | |
| from effgen import __version__ | |
| return __version__ | |
| except Exception: # noqa: BLE001 | |
| return "0.0.0" | |
| def _truthy_env(name: str) -> bool: | |
| """Return True when env var *name* is set to a truthy value.""" | |
| return os.getenv(name, "0").strip().lower() in ("1", "true", "yes", "on") | |
| # --------------------------------------------------------------------------- | |
| # App factory | |
| # --------------------------------------------------------------------------- | |
| def create_app( | |
| *, | |
| dev_mode: bool | None = None, | |
| oidc_issuer: str | None = None, | |
| oidc_client_id: str | None = None, | |
| oidc_jwks_uri: str | None = None, | |
| api_key: str | None = None, | |
| metrics_auth: bool = False, | |
| public_metrics: bool | None = None, | |
| public_dashboard: bool | None = None, | |
| public_playground: bool | None = None, | |
| cors_origins: list[str] | None = None, | |
| rate_limit_per_minute: int | None = None, | |
| trust_proxy: bool | None = None, | |
| runner: Any = None, | |
| extra_models: Any = None, | |
| ) -> Any: | |
| """Create and return the FastAPI application. | |
| All parameters fall back to environment variables when ``None``. | |
| Parameters | |
| ---------- | |
| runner: | |
| Optional callable ``runner(prompt, *, model, tools, stream, ...)`` used | |
| by the OpenAI-compatible ``/v1`` endpoints. When ``None`` a default | |
| agent-backed runner is constructed lazily on first use. | |
| extra_models: | |
| Optional callable returning model ids to list in ``GET /v1/models`` | |
| alongside the legacy aliases. When ``None`` this defaults to the ids | |
| the default runner has actually served a successful response for. | |
| trust_proxy: | |
| Whether the per-IP rate limiter should trust the first | |
| ``X-Forwarded-For`` hop as the client IP. Defaults to ``False`` (the | |
| raw socket peer is used); enable only when the deployment sits behind | |
| a reverse proxy that sets/overwrites this header, since any direct | |
| caller can otherwise set it to bypass the limit. | |
| Args: | |
| dev_mode: Relax CORS for local development. | |
| oidc_issuer: Issuer whose tokens the server accepts. | |
| oidc_client_id: Audience the tokens must carry. | |
| oidc_jwks_uri: Where the issuer's signing keys are fetched from. | |
| api_key: The API key clients authenticate with; one is minted when absent. | |
| metrics_auth: Require authentication on ``/metrics``. | |
| public_metrics: Serve ``/metrics`` without authentication. | |
| public_dashboard: Serve the dashboard without authentication. | |
| public_playground: Serve the playground without authentication. | |
| cors_origins: Origins allowed to call the API cross-origin. | |
| rate_limit_per_minute: Per-client request cap, or ``None`` for no limit. | |
| """ | |
| try: | |
| from fastapi import FastAPI | |
| from fastapi.responses import JSONResponse | |
| except ImportError as exc: | |
| raise ImportError("fastapi is required: pip install 'effgen[server]'") from exc | |
| _dev = dev_mode if dev_mode is not None else (os.getenv("EFFGEN_DEV_MODE", "0") == "1") | |
| # Observability endpoints are protected by default. They become public only | |
| # in dev mode or when an operator explicitly opts in (constructor flag or | |
| # EFFGEN_PUBLIC_METRICS / EFFGEN_PUBLIC_DASHBOARD). The legacy | |
| # ``metrics_auth`` flag (and EFFGEN_METRICS_AUTH) still force metrics auth on. | |
| if public_metrics is None: | |
| public_metrics = _dev or _truthy_env("EFFGEN_PUBLIC_METRICS") | |
| if metrics_auth or _truthy_env("EFFGEN_METRICS_AUTH"): | |
| public_metrics = False | |
| if public_dashboard is None: | |
| public_dashboard = _dev or _truthy_env("EFFGEN_PUBLIC_DASHBOARD") | |
| # The playground drives real, billed model calls, so its local-view opt-in | |
| # is a flag of its own (not shared with the dashboard's) — an operator must | |
| # knowingly authorize spend from a page anyone with the URL can reach. | |
| if public_playground is None: | |
| public_playground = _dev or _truthy_env("EFFGEN_PUBLIC_PLAYGROUND") | |
| app = FastAPI( | |
| title="effGen API", | |
| description="effGen — Efficient Agent Framework API", | |
| version=_server_version(), | |
| docs_url="/docs", | |
| redoc_url="/redoc", | |
| ) | |
| # Every error this app returns — validation (422), a raised HTTPException | |
| # (including the router's own 404/405), and an unhandled exception (500) — | |
| # is rendered with the same `{"error": {message, type, param, code}}` | |
| # envelope the model routes use, so a client branches on `err.type` / | |
| # `err.code` uniformly instead of switching between that and FastAPI's | |
| # `{"detail": ...}`. | |
| try: | |
| from fastapi.exceptions import RequestValidationError | |
| from starlette.exceptions import HTTPException as _StarletteHTTPException | |
| from effgen.api.openai_compat import _classify_http, error_envelope | |
| async def _validation_handler(request: Any, exc: RequestValidationError) -> Any: | |
| errs = exc.errors() | |
| try: | |
| first = errs[0] | |
| loc = ".".join(str(p) for p in first.get("loc", []) if p != "body") | |
| msg = f"{loc}: {first.get('msg')}" if loc else str(first.get("msg")) | |
| except Exception: # noqa: BLE001 - fall back to a generic message | |
| msg = "invalid request body" | |
| return JSONResponse( | |
| status_code=422, | |
| content=error_envelope(422, msg, code="invalid_request_body", redact=False), | |
| ) | |
| async def _http_exception_handler(request: Any, exc: Any) -> Any: | |
| detail = exc.detail if isinstance(exc.detail, str) else "request failed" | |
| # A 404/405 from the router is about the URL, not about a model, so | |
| # it is typed as an invalid request. Response headers the exception | |
| # carries are part of the HTTP contract (``Allow`` on a 405, | |
| # ``WWW-Authenticate`` on a 401) and survive the re-render. | |
| return JSONResponse( | |
| status_code=exc.status_code, | |
| content=error_envelope( | |
| exc.status_code, | |
| detail, | |
| code=_HTTP_ERROR_CODES.get(exc.status_code), | |
| error_type=( | |
| "invalid_request_error" if exc.status_code in (404, 405) else None | |
| ), | |
| ), | |
| headers=getattr(exc, "headers", None), | |
| ) | |
| async def _unhandled_handler(request: Any, exc: Exception) -> Any: | |
| # The handler runs outside the ``except`` block, so attach the | |
| # exception explicitly to keep the traceback in the log. | |
| logger.error( | |
| "Unhandled error serving %s", | |
| getattr(request, "url", ""), | |
| exc_info=exc, | |
| ) | |
| status, _err_type, code = _classify_http(exc) | |
| if status < 500: | |
| status, code = 500, None | |
| # The message is redacted before it leaves the process so an | |
| # upstream error carrying a key does not reach the client. | |
| return JSONResponse( | |
| status_code=status, | |
| content=error_envelope(status, str(exc) or exc.__class__.__name__, code=code), | |
| ) | |
| except Exception: # noqa: BLE001 - error handlers are best-effort | |
| logger.debug("Could not install error handlers", exc_info=True) | |
| # ------------------------------------------------------------------ | |
| # Middleware stack. ``add_middleware`` is LIFO, so the *last* added wraps | |
| # as the outermost layer. We want, outer → inner: | |
| # | |
| # production (CORS/gzip/request-id) → audit → auth → RBAC/budget → route | |
| # | |
| # Auth must run before RBAC/budget so request.state.user is populated, and | |
| # RBAC/budget sits innermost (just outside the route) so its request-body | |
| # replay survives the production BaseHTTPMiddleware layers above it. | |
| # ------------------------------------------------------------------ | |
| # 1. RBAC + budget enforcement (innermost; reads body for /v1 endpoints) | |
| app.add_middleware(RBACBudgetMiddleware) # type: ignore[arg-type] | |
| # 1b. Body-size cap for body-accepting routes RBAC/budget doesn't cover | |
| # (e.g. /v1/embeddings, which needs no RBAC/budget enforcement but must | |
| # still bound how much a client can make the server buffer). | |
| app.add_middleware(MaxBodySizeMiddleware) # type: ignore[arg-type] | |
| # 2. Auth middleware (validates JWT, populates request.state.user) | |
| from effgen.server.auth import AuthMiddleware | |
| app.add_middleware( | |
| AuthMiddleware, # type: ignore[arg-type] | |
| issuer=oidc_issuer, | |
| client_id=oidc_client_id, | |
| jwks_uri=oidc_jwks_uri, | |
| api_key=api_key, | |
| public_metrics=public_metrics, | |
| public_dashboard=public_dashboard, | |
| public_playground=public_playground, | |
| dev_mode=_dev, | |
| ) | |
| # Record the resolved playground settings so the playground bootstrap route | |
| # can decide whether to expose a local-view session key and report whether | |
| # spend is authorized without re-reading the environment. | |
| app.state.public_playground = bool(public_playground) | |
| app.state.dev_mode = bool(_dev) | |
| app.state.api_key = api_key if api_key is not None else os.getenv("EFFGEN_API_KEY", "") | |
| # 3. Audit middleware (logs every request; reads user from state set by auth) | |
| from effgen.server.audit import AuditMiddleware | |
| app.add_middleware(AuditMiddleware) # type: ignore[arg-type] | |
| # 4. Production middleware (CORS, gzip, request-ID) — outermost. | |
| # CORS is fail-closed: outside dev mode, cross-origin access is disabled | |
| # unless ``cors_origins`` (or EFFGEN_CORS_ORIGINS) is explicitly configured. | |
| if cors_origins is None: | |
| _env_origins = os.getenv("EFFGEN_CORS_ORIGINS", "").strip() | |
| if _env_origins: | |
| cors_origins = [o.strip() for o in _env_origins.split(",") if o.strip()] | |
| try: | |
| from effgen.api.middleware import install_production_middleware | |
| install_production_middleware( | |
| app, | |
| cors_origins=cors_origins, | |
| dev_mode=_dev, | |
| rate_limit_per_minute=rate_limit_per_minute, | |
| trust_proxy=trust_proxy, | |
| ) | |
| except ImportError: | |
| logger.warning("Production middleware not available") | |
| # ------------------------------------------------------------------ | |
| # Routes | |
| # ------------------------------------------------------------------ | |
| async def health() -> dict[str, str]: | |
| """Liveness probe — always public. | |
| ``/health`` is the canonical name; ``/healthz`` and ``/livez`` are | |
| aliases for the conventional Kubernetes liveness probe path. A 200 here | |
| means the process is up and serving requests. | |
| """ | |
| return {"status": "ok", "version": _server_version()} | |
| async def ready() -> dict[str, str]: | |
| """Readiness probe — always public. | |
| Returns 200 once the application is constructed and able to accept | |
| traffic. K8s readiness probes conventionally use ``/readyz``; ``/ready`` | |
| is provided as an alias. Kept lightweight (no per-request model load) so | |
| a load balancer can poll it cheaply. | |
| """ | |
| return {"status": "ready", "version": _server_version()} | |
| async def metrics(request: _FastAPIRequest) -> Any: # type: ignore[valid-type] | |
| """Prometheus metrics (requires auth by default; EFFGEN_PUBLIC_METRICS=1 opens it).""" | |
| try: | |
| from prometheus_client import CONTENT_TYPE_LATEST, generate_latest | |
| from starlette.responses import Response | |
| payload = generate_latest().decode("utf-8", errors="replace") | |
| try: | |
| from effgen.observability.metrics import export_metrics | |
| payload += "\n" + export_metrics() | |
| except Exception: # noqa: BLE001 - metrics export is optional; omit it on failure | |
| pass | |
| return Response(content=payload, media_type=CONTENT_TYPE_LATEST) | |
| except ImportError: | |
| from effgen.api.openai_compat import error_envelope | |
| return JSONResponse( | |
| error_envelope( | |
| 503, | |
| "Metrics are unavailable: prometheus_client is not installed. " | |
| "Install it with `pip install 'effgen[server]'`.", | |
| code="metrics_unavailable", | |
| redact=False, | |
| ), | |
| status_code=503, | |
| ) | |
| async def slo_status() -> dict[str, Any]: | |
| """SLO burn-rate status for every registered SLO (public, like /health). | |
| Lists objectives this process registered via | |
| ``effgen.observability.slo.get_tracker().register(...)`` and recorded | |
| events against with ``tracker.record(name, ok=...)``. It does not | |
| compute latency or availability from request metrics, so it stays empty | |
| on a server that has served traffic but registered no objective; the | |
| measured percentiles and availability for that traffic are in the | |
| ``slo`` block of ``/dashboard/data.json``. An empty list carries a | |
| ``detail`` note saying so. | |
| """ | |
| from effgen.observability.slo import EMPTY_SLO_DETAIL, get_tracker | |
| statuses = get_tracker().all_statuses() | |
| payload: dict[str, Any] = {"slos": statuses} | |
| if not statuses: | |
| payload["detail"] = EMPTY_SLO_DETAIL | |
| return payload | |
| async def whoami(request: _FastAPIRequest) -> dict[str, Any]: # type: ignore[valid-type] | |
| """Return the current principal's identity (requires auth in non-dev mode).""" | |
| user = getattr(request.state, "user", None) | |
| if user is None: | |
| return {"principal": "anonymous"} | |
| return { | |
| "sub": user.sub, | |
| "iss": user.iss, | |
| "roles": user.roles, | |
| "email": user.email, | |
| } | |
| async def rbac_policy(request: _FastAPIRequest) -> dict[str, Any]: # type: ignore[valid-type] | |
| """Return the effective RBAC policy for the current principal.""" | |
| from effgen.api.openai_compat import error_envelope | |
| from effgen.server.rbac import PolicyDenied, resolve_policy | |
| user = getattr(request.state, "user", None) | |
| roles: list[str] = getattr(user, "roles", []) if user else [] | |
| try: | |
| policy = resolve_policy(roles) | |
| except PolicyDenied as exc: | |
| # Same envelope the RBAC middleware returns when it denies a model | |
| # call, so both denials read identically to a client. | |
| return JSONResponse( | |
| error_envelope(exc.status_code, str(exc), redact=False), | |
| status_code=exc.status_code, | |
| ) | |
| return { | |
| "roles": [r.name for r in policy.roles], | |
| "allowed_tools": sorted(policy.allowed_tools) or ["*"], | |
| "allowed_models": sorted(policy.allowed_models) or ["*"], | |
| "max_cost_per_day": policy.max_cost_per_day, | |
| } | |
| async def rbac_roles() -> list[dict[str, Any]]: | |
| """List all defined RBAC roles.""" | |
| from effgen.server.rbac import list_roles | |
| return [r.to_dict() for r in list_roles()] | |
| # Import and mount the OpenAI-compatible + embeddings routers, with RBAC | |
| # and per-principal cost-cap enforcement layered on top. | |
| _mount_existing_routers(app, runner=runner, extra_models=extra_models) | |
| # Mount the local dashboard (static SPA + data.json + SSE spans). | |
| _mount_dashboard(app) | |
| # Mount the in-browser playground (static SPA + bootstrap config). | |
| _mount_playground(app) | |
| return app | |
Xet Storage Details
- Size:
- 20.3 kB
- Xet hash:
- 2fe88d4668ed465cb202f42103c0a906613f686c414f9e5e4234c148ad0441fa
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.