Buckets:
AnonymousSpottedToad/human_verified_bucket / codebase /python /ctrl-gaurav_effGen /effgen /server /app_runner.py
| """Default agent-backed runner and model pool for the effGen API server. | |
| Provides the runner the OpenAI-compatible ``/v1`` endpoints use when | |
| :func:`effgen.server.app.create_app` is not given one: it drives an | |
| :class:`~effgen.core.agent.Agent` around a pooled model, normalizes | |
| OpenAI-style ``provider/model`` ids for effGen's loader, resolves tool specs | |
| against the built-in registry, and mounts the ``/v1`` routers. | |
| """ | |
| from __future__ import annotations | |
| import logging | |
| import os | |
| import threading | |
| from collections import OrderedDict | |
| from typing import Any | |
| from effgen.errors import RunStoppedError | |
| logger = logging.getLogger(__name__) | |
| # Known provider prefixes — stable fallback when the dynamic ProviderRegistry | |
| # is unavailable or has been reset (e.g. in test teardowns). | |
| _KNOWN_PROVIDERS: frozenset[str] = frozenset( | |
| { | |
| "cerebras", | |
| "openai", | |
| "anthropic", | |
| "gemini", | |
| "groq", | |
| "together", | |
| "fireworks", | |
| "replicate", | |
| "hf", | |
| } | |
| ) | |
| # --------------------------------------------------------------------------- | |
| # Model pool — reuse loaded models across requests | |
| # --------------------------------------------------------------------------- | |
| # | |
| # Constructing a fresh Agent per request is cheap, but each Agent used to spin | |
| # up its own ModelLoader and *reload the model* every call — catastrophic for | |
| # local GPU models and wasteful for cloud adapters. We instead pool the loaded | |
| # *model* (the heavy object) keyed by its resolved id, bounded by an LRU, and | |
| # build a lightweight, fresh-memory Agent around the pooled model per request. | |
| # A fresh Agent per call keeps the endpoint stateless (no conversation bleed | |
| # between unrelated API requests) while paying the model-load cost only once. | |
| _MODEL_POOL: "OrderedDict[str, Any]" = OrderedDict() | |
| _MODEL_POOL_LOCK = threading.Lock() | |
| _LOADING_LOCKS: dict[str, Any] = {} | |
| # Ids that produced at least one successful response this run. This drives the | |
| # ``GET /v1/models`` listing so it advertises only models the server has really | |
| # served — not every id that merely loaded a cloud adapter (a nonexistent | |
| # ``provider:model`` can construct an adapter and only 404 on the actual call). | |
| _SERVED_MODEL_IDS: "OrderedDict[str, None]" = OrderedDict() | |
| _SERVED_MODEL_LOCK = threading.Lock() | |
| _SERVED_MODEL_MAX = 64 | |
| def _record_served_model(resolved_model: str) -> None: | |
| """Record that *resolved_model* returned a successful response.""" | |
| if not resolved_model: | |
| return | |
| with _SERVED_MODEL_LOCK: | |
| _SERVED_MODEL_IDS[resolved_model] = None | |
| _SERVED_MODEL_IDS.move_to_end(resolved_model) | |
| while len(_SERVED_MODEL_IDS) > _SERVED_MODEL_MAX: | |
| _SERVED_MODEL_IDS.popitem(last=False) | |
| def _served_model_ids() -> list[str]: | |
| """Return the ids that have served a successful response this run.""" | |
| with _SERVED_MODEL_LOCK: | |
| return list(_SERVED_MODEL_IDS.keys()) | |
| def _model_pool_max() -> int: | |
| try: | |
| return max(1, int(os.getenv("EFFGEN_MODEL_POOL_SIZE", "4"))) | |
| except ValueError: | |
| return 4 | |
| def _get_pooled_model(resolved_model: str) -> Any: | |
| """Return a loaded model for *resolved_model*, reusing a pooled instance. | |
| Thread-safe with a bounded LRU. The (potentially slow) load happens outside | |
| the pool lock, serialized per-model so two concurrent first requests don't | |
| both load the same model. | |
| """ | |
| with _MODEL_POOL_LOCK: | |
| model = _MODEL_POOL.get(resolved_model) | |
| if model is not None: | |
| _MODEL_POOL.move_to_end(resolved_model) | |
| return model | |
| load_lock = _LOADING_LOCKS.setdefault(resolved_model, threading.Lock()) | |
| with load_lock: | |
| # Re-check: another thread may have loaded it while we waited. | |
| with _MODEL_POOL_LOCK: | |
| model = _MODEL_POOL.get(resolved_model) | |
| if model is not None: | |
| _MODEL_POOL.move_to_end(resolved_model) | |
| return model | |
| from effgen.models.model_loader import load_model as _load_model | |
| model = _load_model(resolved_model) | |
| evicted: list[Any] = [] | |
| with _MODEL_POOL_LOCK: | |
| _MODEL_POOL[resolved_model] = model | |
| _MODEL_POOL.move_to_end(resolved_model) | |
| while len(_MODEL_POOL) > _model_pool_max(): | |
| _, old = _MODEL_POOL.popitem(last=False) | |
| evicted.append(old) | |
| _LOADING_LOCKS.pop(resolved_model, None) | |
| for old in evicted: | |
| for closer in ("close", "unload", "shutdown"): | |
| fn = getattr(old, closer, None) | |
| if callable(fn): | |
| try: | |
| fn() | |
| except Exception: # noqa: BLE001 | |
| logger.debug("Error closing evicted model", exc_info=True) | |
| break | |
| return model | |
| def _extract_usage(response: Any) -> tuple[int | None, int | None]: | |
| """Pull (prompt_tokens, completion_tokens) from an AgentResponse. | |
| Prefers provider/tokenizer counts the agent recorded in metadata; falls | |
| back to the aggregate ``tokens_used`` for completion tokens. | |
| """ | |
| meta = getattr(response, "metadata", None) or {} | |
| def _int_or_none(v: Any) -> int | None: | |
| try: | |
| return int(v) if v is not None else None | |
| except (TypeError, ValueError): | |
| return None | |
| prompt = _int_or_none(meta.get("input_tokens", meta.get("prompt_tokens"))) | |
| completion = _int_or_none(meta.get("output_tokens", meta.get("completion_tokens"))) | |
| if completion is None: | |
| completion = _int_or_none(getattr(response, "tokens_used", None)) or None | |
| return prompt, completion | |
| def _extract_cost(response: Any) -> float | None: | |
| """Pull per-call ``cost_usd`` from an AgentResponse, if the model is priced. | |
| effGen records the dollar cost of a run in ``metadata["cost_usd"]`` (from the | |
| provider's pricing table). Returns ``None`` for unpriced/local models so the | |
| response can omit cost rather than report a misleading zero. | |
| """ | |
| meta = getattr(response, "metadata", None) or {} | |
| raw = meta.get("cost_usd") | |
| try: | |
| return float(raw) if raw is not None else None | |
| except (TypeError, ValueError): | |
| return None | |
| _TRACE_ARG_MAX = 200 | |
| _TRACE_RESULT_MAX = 200 | |
| def _extract_tool_trace(response: Any) -> list[dict[str, Any]]: | |
| """Summarize the tools a run executed, in call order. | |
| The agent records each tool call as a start/complete (or failed) pair of | |
| events in ``response.execution_trace``. This pairs them into a compact | |
| per-tool summary — ``{tool, args, result_summary, ok, duration_ms}`` — so a | |
| caller can render what the agent actually did. Returns an empty list when no | |
| tool ran (a direct model answer). | |
| """ | |
| trace = getattr(response, "execution_trace", None) or [] | |
| if not isinstance(trace, list): | |
| return [] | |
| steps: list[dict[str, Any]] = [] | |
| pending: dict[str, Any] | None = None | |
| def _truncate(value: Any, limit: int) -> str: | |
| text = "" if value is None else str(value) | |
| return text if len(text) <= limit else text[: limit - 1] + "…" | |
| for event in trace: | |
| if not isinstance(event, dict): | |
| continue | |
| etype = event.get("type") | |
| data = event.get("data") or {} | |
| if etype == "tool_call_start": | |
| if pending is not None: | |
| steps.append(pending) # a start with no matching completion | |
| pending = { | |
| "tool": data.get("tool_name") or "tool", | |
| "args": _truncate(data.get("tool_input"), _TRACE_ARG_MAX), | |
| "result_summary": None, | |
| "ok": None, | |
| "duration_ms": None, | |
| "_start_ts": event.get("timestamp"), | |
| } | |
| elif etype in ("tool_call_complete", "tool_call_failed"): | |
| step = pending or { | |
| "tool": data.get("tool_name") or "tool", | |
| "args": _truncate(data.get("input"), _TRACE_ARG_MAX), | |
| "_start_ts": None, | |
| } | |
| pending = None | |
| ok = etype == "tool_call_complete" | |
| if ok: | |
| step["result_summary"] = _truncate(data.get("result"), _TRACE_RESULT_MAX) | |
| else: | |
| step["result_summary"] = _truncate(data.get("error"), _TRACE_RESULT_MAX) | |
| step["ok"] = ok | |
| start_ts = step.pop("_start_ts", None) | |
| end_ts = event.get("timestamp") | |
| if isinstance(start_ts, int | float) and isinstance(end_ts, int | float): | |
| step["duration_ms"] = round(max(0.0, (end_ts - start_ts) * 1000), 1) | |
| step.setdefault("result_summary", None) | |
| step.setdefault("ok", ok) | |
| step.setdefault("duration_ms", None) | |
| steps.append(step) | |
| if pending is not None: | |
| pending.pop("_start_ts", None) | |
| steps.append(pending) | |
| return steps | |
| class _StreamWithUsage: | |
| """Token iterator that publishes the run's usage once it is exhausted. | |
| The OpenAI-compatible route reads ``usage`` after the last token to fill the | |
| ``stream_options.include_usage`` chunk, so a streamed request reports the | |
| same token counts and cost a non-streamed one does. It also reads | |
| ``finish_reason``, which is ``"length"`` when the loop stopped the run before | |
| the model wrote an answer — the same thing the non-streamed path reports. | |
| Iterating closes the ephemeral agent (releasing its memory handles and | |
| circuit breakers, but not the pooled model, which is shared and stays | |
| loaded). | |
| """ | |
| def __init__(self, agent: Any, prompt: str, resolved_model: str) -> None: | |
| self._agent = agent | |
| self._prompt = prompt | |
| self._resolved_model = resolved_model | |
| self.usage: dict[str, Any] | None = None | |
| self.finish_reason: str = "stop" | |
| def __iter__(self) -> Any: | |
| served = False | |
| try: | |
| for chunk in self._agent.stream(self._prompt): | |
| served = True | |
| yield chunk | |
| self.usage = self._agent.last_stream_usage | |
| record = getattr(self._agent, "last_stream_response", None) | |
| if getattr(record, "outcome", None) == "stopped": | |
| self.finish_reason = "length" | |
| finally: | |
| self._agent.close() | |
| if served: | |
| _record_served_model(self._resolved_model) | |
| def _build_default_runner() -> Any: | |
| """Construct an agent-backed runner for the OpenAI-compatible endpoints. | |
| Returns a callable ``runner(prompt, *, model, tools, stream, **kw)`` that | |
| drives an :class:`~effgen.core.agent.Agent` around a *pooled* model. For | |
| ``stream=True`` it returns a lazy token generator (so the route can begin | |
| flushing SSE immediately); otherwise it returns a | |
| :class:`~effgen.api.openai_compat.RunnerResult` carrying real usage and the | |
| resolved model id. | |
| """ | |
| def _runner( | |
| prompt: str, | |
| *, | |
| model: str, | |
| tools: Any = None, | |
| stream: bool = False, | |
| temperature: float | None = None, | |
| **_: Any, | |
| ) -> Any: | |
| from effgen.api.openai_compat import RunnerResult | |
| from effgen.core.agent import Agent, AgentConfig | |
| # Pool access resolves through ``effgen.server.app`` at call time so a | |
| # caller that rebinds ``app._get_pooled_model`` (e.g. to supply a stub | |
| # model) stays effective. | |
| from effgen.server import app as _app_module | |
| resolved_model = _normalize_model_id(model) | |
| pooled_model = _app_module._get_pooled_model(resolved_model) | |
| resolved_tools = _resolve_tools(tools) | |
| config = AgentConfig( | |
| # Name the served model in the agent's identity so a server-driven | |
| # run is distinguishable in traces and run history instead of every | |
| # request reading as the same "api" agent. | |
| name=f"api:{resolved_model}", | |
| model=pooled_model, # pre-loaded instance → no per-request reload | |
| tools=resolved_tools, | |
| temperature=temperature if temperature is not None else 0.7, | |
| require_model=True, | |
| # A generation failure must reach the route as a typed exception so | |
| # it maps to a real HTTP status (404/502/503/...) via the same | |
| # taxonomy every other runner failure uses, instead of returning | |
| # HTTP 200 with the error text as the answer. ``stream()`` already | |
| # raises on failure unconditionally, so this only changes run(). | |
| raise_on_error=True, | |
| ) | |
| agent = Agent(config) | |
| if stream: | |
| # Close the ephemeral agent once the stream is exhausted. ``close()`` | |
| # releases the agent's own resources (memory handles, circuit | |
| # breakers) but NOT the pooled model, which is shared and stays | |
| # loaded — and it silences the "garbage-collected without close()" | |
| # warning that would otherwise fire per streamed request. | |
| return _StreamWithUsage(agent, prompt, resolved_model) | |
| try: | |
| # A failure raises here (raise_on_error=True above), so a response | |
| # that reaches this point is always a success — except for a run the | |
| # loop stopped, which is served below as a completed request whose | |
| # generation was cut short. | |
| try: | |
| response = agent.run(prompt) | |
| finish_reason = "stop" | |
| except RunStoppedError as stopped: | |
| # The request was valid and the server did its job; the model | |
| # did not finish. That is a 200 with OpenAI's own vocabulary for | |
| # "cut off by a limit", not a 5xx blaming the server. | |
| response = stopped.response | |
| finish_reason = "length" | |
| _record_served_model(resolved_model) | |
| prompt_tokens, completion_tokens = _extract_usage(response) | |
| tool_trace = _extract_tool_trace(response) | |
| run_meta = getattr(response, "metadata", None) or {} | |
| # Surface which tools ran (and how many) so a caller can render the | |
| # step trace. Lives in the runner's ``metadata`` and is carried into | |
| # the response's non-standard ``effgen`` object by the OpenAI-compat | |
| # layer; standard OpenAI clients ignore unknown keys. | |
| extra_meta: dict[str, Any] = { | |
| "tool_calls": int(getattr(response, "tool_calls", 0) or 0), | |
| } | |
| if tool_trace: | |
| extra_meta["trace"] = tool_trace | |
| run_id = run_meta.get("run_id") | |
| if run_id: | |
| extra_meta["run_id"] = run_id | |
| stop_reason = getattr(response, "stop_reason", None) | |
| if stop_reason: | |
| extra_meta["stop_reason"] = stop_reason | |
| outcome = getattr(response, "outcome", None) | |
| if outcome: | |
| extra_meta["outcome"] = outcome | |
| partial = getattr(response, "partial", None) | |
| if partial is not None: | |
| extra_meta["partial"] = partial.to_dict() | |
| return RunnerResult( | |
| text=getattr(response, "output", "") or "", | |
| prompt_tokens=prompt_tokens, | |
| completion_tokens=completion_tokens, | |
| resolved_model=resolved_model, | |
| cost_usd=_extract_cost(response), | |
| finish_reason=finish_reason, | |
| metadata=extra_meta, | |
| ) | |
| finally: | |
| agent.close() | |
| return _runner | |
| def _normalize_model_id(model: str) -> str: | |
| """Normalize an OpenAI-style ``provider/model`` id for effGen's loader. | |
| OpenAI-compatible clients send ``"cerebras/llama3.1-8b"`` (slash), but | |
| effGen's ``ModelLoader`` routes API providers via the ``"provider:model"`` | |
| (colon) prefix. Without this, a slash id falls through to the local | |
| Transformers path and fails with "not a valid model identifier". Rewrite | |
| the *provider* separator to a colon only when the prefix is a known | |
| provider, leaving bare ids and HF ``org/repo`` ids untouched. | |
| """ | |
| if not isinstance(model, str) or "/" not in model: | |
| return model | |
| prefix, _, rest = model.partition("/") | |
| try: | |
| from effgen.models.registry import ProviderRegistry | |
| providers = ProviderRegistry.list_providers() | |
| if not providers: | |
| # Registry may have been reset (e.g. by test teardowns). | |
| # Fall back to the static provider set rather than attempting a | |
| # re-import that could cause circular-import or side-effect issues. | |
| providers = _KNOWN_PROVIDERS # type: ignore[assignment] | |
| if prefix in providers: | |
| return f"{prefix}:{rest}" | |
| except Exception: # noqa: BLE001 - registry optional; fall back to static set | |
| if prefix in _KNOWN_PROVIDERS: | |
| return f"{prefix}:{rest}" | |
| return model | |
| def _resolve_tools(tools: Any) -> list[Any]: | |
| """Resolve OpenAI-style tool specs / bare names into effGen tool objects. | |
| The OpenAI chat schema sends tools as ``{"type": "function", "function": | |
| {"name": ...}}`` dicts; effGen's :class:`Agent` expects tool *instances*. | |
| Names are resolved against the built-in tool registry (RBAC has already | |
| authorized the names). A requested tool that the server does not host is | |
| **not** silently dropped — that would leave a client expecting OpenAI | |
| function-calling with prose and no ``tool_calls``. Instead the unhosted | |
| names are collected and surfaced as an :class:`UnknownToolError`, which the | |
| route turns into a clear ``400``. | |
| """ | |
| if not tools: | |
| return [] | |
| import asyncio | |
| from effgen.api.openai_compat import UnknownToolError | |
| from effgen.tools.registry import ToolRegistry | |
| registry = ToolRegistry() | |
| try: | |
| registry.discover_builtin_tools() | |
| except Exception: # noqa: BLE001 - tool discovery is best-effort at startup; server still serves | |
| pass | |
| def _get(name: str) -> Any: | |
| """Resolve one tool name, working whether or not a loop is running.""" | |
| coro = registry.get_tool(name) | |
| try: | |
| asyncio.get_running_loop() | |
| except RuntimeError: | |
| return asyncio.run(coro) | |
| # A loop is already running (FastAPI route): run in a worker thread. | |
| import concurrent.futures | |
| with concurrent.futures.ThreadPoolExecutor(max_workers=1) as ex: | |
| return ex.submit(lambda: asyncio.run(coro)).result() | |
| resolved: list[Any] = [] | |
| unresolved: list[str] = [] | |
| for tool in tools: | |
| if not isinstance(tool, str) and not isinstance(tool, dict): | |
| # Already a tool object. | |
| resolved.append(tool) | |
| continue | |
| name = _tool_name(tool) | |
| try: | |
| obj = _get(name) | |
| except Exception as exc: # noqa: BLE001 | |
| logger.warning("Could not resolve tool %r: %s", name, exc) | |
| obj = None | |
| if obj is not None: | |
| resolved.append(obj) | |
| else: | |
| unresolved.append(name) | |
| if unresolved: | |
| raise UnknownToolError(unresolved) | |
| return resolved | |
| def _tool_name(tool: Any) -> str: | |
| """Extract a tool name from an OpenAI-style tool spec or a bare string.""" | |
| if isinstance(tool, dict): | |
| return tool.get("function", {}).get("name") or tool.get("name") or "tool" | |
| return str(tool) | |
| def _mount_existing_routers( | |
| app: Any, *, runner: Any = None, extra_models: Any = None | |
| ) -> None: | |
| """Mount the OpenAI-compat + embeddings routers.""" | |
| try: | |
| from effgen.api.openai_compat import create_openai_router | |
| _runner = runner or _build_default_runner() | |
| router = create_openai_router( | |
| _runner, extra_models=extra_models or _served_model_ids | |
| ) | |
| app.include_router(router) | |
| logger.info("Mounted OpenAI-compat router at /v1 (RBAC + budget enforced)") | |
| except Exception as exc: # noqa: BLE001 | |
| logger.warning("OpenAI-compat router not mounted: %s", exc) | |
| try: | |
| from effgen.api.embeddings import create_embeddings_router | |
| router = create_embeddings_router() | |
| # Router already has /v1 in path; mount at root | |
| app.include_router(router) | |
| logger.info("Mounted embeddings router") | |
| except Exception as exc: # noqa: BLE001 | |
| logger.debug("Embeddings router not mounted: %s", exc) | |
Xet Storage Details
- Size:
- 20.5 kB
- Xet hash:
- 92ab8e6c4173247be4ae8a5d6e908d6263aa6683a6993f2685333dc9388635d2
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.