Add loan-advisor-agent (part 2)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/ARCHITECTURE.md +292 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/handler.py +1065 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/tools.py +182 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/transformation.py +373 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/weights_biases.py +219 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/__init__.py +68 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/http_handler.py +697 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/__init__.py +15 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/handler.py +155 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/streaming_iterator.py +286 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/transformation.py +299 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/main.py +644 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/streaming_iterator.py +271 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/utils.py +87 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/README.md +13 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/api_route_to_call_types.py +69 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/app_crypto.py +33 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/asyncify.py +116 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/audio_utils/audio_health_check.wav +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/audio_utils/utils.py +275 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/cached_imports.py +59 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/cli_token_utils.py +58 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/core_helpers.py +437 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/coroutine_checker.py +64 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/credential_accessor.py +35 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/custom_logger_registry.py +184 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/dd_tracing.py +123 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/default_encoding.py +52 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/dot_notation_indexing.py +255 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/duration_parser.py +391 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/env_utils.py +21 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/fallback_utils.py +80 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_blog_posts.py +164 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_litellm_params.py +164 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py +958 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_model_cost_map.py +299 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_provider_specific_headers.py +29 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_supported_openai_params.py +320 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/health_check_helpers.py +219 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/health_check_utils.py +28 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +75 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/json_validation_rule.py +133 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/litellm_logging.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +729 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +38 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/utils.py +1034 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_request_utils.py +84 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +833 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_response_utils/get_api_base.py +119 -0
loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/ARCHITECTURE.md
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# WebSearch Interception Architecture
|
| 2 |
+
|
| 3 |
+
Server-side WebSearch tool execution for models that don't natively support it (e.g., Bedrock/Claude).
|
| 4 |
+
|
| 5 |
+
## How It Works
|
| 6 |
+
|
| 7 |
+
User makes **ONE** `litellm.messages.acreate()` call → Gets final answer with search results.
|
| 8 |
+
The agentic loop happens transparently on the server.
|
| 9 |
+
|
| 10 |
+
## LiteLLM Standard Web Search Tool
|
| 11 |
+
|
| 12 |
+
LiteLLM defines a standard web search tool format (`litellm_web_search`) that all native provider tools are converted to. This enables consistent interception across providers.
|
| 13 |
+
|
| 14 |
+
**Standard Tool Definition** (defined in `tools.py`):
|
| 15 |
+
```python
|
| 16 |
+
{
|
| 17 |
+
"name": "litellm_web_search",
|
| 18 |
+
"description": "Search the web for information...",
|
| 19 |
+
"input_schema": {
|
| 20 |
+
"type": "object",
|
| 21 |
+
"properties": {
|
| 22 |
+
"query": {"type": "string", "description": "The search query"}
|
| 23 |
+
},
|
| 24 |
+
"required": ["query"]
|
| 25 |
+
}
|
| 26 |
+
}
|
| 27 |
+
```
|
| 28 |
+
|
| 29 |
+
**Tool Name Constant**: `LITELLM_WEB_SEARCH_TOOL_NAME = "litellm_web_search"` (defined in `litellm/constants.py`)
|
| 30 |
+
|
| 31 |
+
### Supported Tool Formats
|
| 32 |
+
|
| 33 |
+
The interception system automatically detects and handles:
|
| 34 |
+
|
| 35 |
+
| Tool Format | Example | Provider | Detection Method | Future-Proof |
|
| 36 |
+
|-------------|---------|----------|------------------|-------------|
|
| 37 |
+
| **LiteLLM Standard** | `name="litellm_web_search"` | Any | Direct name match | N/A |
|
| 38 |
+
| **Anthropic Native** | `type="web_search_20250305"` | Bedrock, Claude API | Type prefix: `startswith("web_search_")` | ✅ Yes (web_search_2026, etc.) |
|
| 39 |
+
| **Claude Code CLI** | `name="web_search"`, `type="web_search_20250305"` | Claude Code | Name + type check | ✅ Yes (version-agnostic) |
|
| 40 |
+
| **Legacy** | `name="WebSearch"` | Custom | Name match | N/A (backwards compat) |
|
| 41 |
+
|
| 42 |
+
**Future Compatibility**: The `startswith("web_search_")` check in `tools.py` automatically supports future Anthropic web search versions.
|
| 43 |
+
|
| 44 |
+
### Claude Code CLI Integration
|
| 45 |
+
|
| 46 |
+
Claude Code (Anthropic's official CLI) sends web search requests using Anthropic's native tool format:
|
| 47 |
+
|
| 48 |
+
```python
|
| 49 |
+
{
|
| 50 |
+
"type": "web_search_20250305",
|
| 51 |
+
"name": "web_search",
|
| 52 |
+
"max_uses": 8
|
| 53 |
+
}
|
| 54 |
+
```
|
| 55 |
+
|
| 56 |
+
**What Happens:**
|
| 57 |
+
1. Claude Code sends native `web_search_20250305` tool to LiteLLM proxy
|
| 58 |
+
2. LiteLLM intercepts and converts to `litellm_web_search` standard format
|
| 59 |
+
3. Bedrock receives converted tool (NOT native format)
|
| 60 |
+
4. Model returns `tool_use` block for `litellm_web_search` (not `server_tool_use`)
|
| 61 |
+
5. LiteLLM's agentic loop intercepts the `tool_use`
|
| 62 |
+
6. Executes `litellm.asearch()` using configured provider (Perplexity, Tavily, etc.)
|
| 63 |
+
7. Returns final answer to Claude Code user
|
| 64 |
+
|
| 65 |
+
**Without Interception**: Bedrock would receive native tool → try to execute natively → return `web_search_tool_result_error` with `invalid_tool_input`
|
| 66 |
+
|
| 67 |
+
**With Interception**: LiteLLM converts → Bedrock returns tool_use → LiteLLM executes search → Returns final answer ✅
|
| 68 |
+
|
| 69 |
+
### Native Tool Conversion
|
| 70 |
+
|
| 71 |
+
Native tools are converted to LiteLLM standard format **before** sending to the provider:
|
| 72 |
+
|
| 73 |
+
1. **Conversion Point** (`litellm/llms/anthropic/experimental_pass_through/messages/handler.py`):
|
| 74 |
+
- In `anthropic_messages()` function (lines 60-127)
|
| 75 |
+
- Runs BEFORE the API request is made
|
| 76 |
+
- Detects native web search tools using `is_web_search_tool()`
|
| 77 |
+
- Converts to `litellm_web_search` format using `get_litellm_web_search_tool()`
|
| 78 |
+
- Prevents provider from executing search natively (avoids `web_search_tool_result_error`)
|
| 79 |
+
|
| 80 |
+
2. **Response Detection** (`transformation.py`):
|
| 81 |
+
- Detects `tool_use` blocks with any web search tool name
|
| 82 |
+
- Handles: `litellm_web_search`, `WebSearch`, `web_search`
|
| 83 |
+
- Extracts search queries for execution
|
| 84 |
+
|
| 85 |
+
**Example Conversion**:
|
| 86 |
+
```python
|
| 87 |
+
# Input (Claude Code's native tool)
|
| 88 |
+
{
|
| 89 |
+
"type": "web_search_20250305",
|
| 90 |
+
"name": "web_search",
|
| 91 |
+
"max_uses": 8
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
# Output (LiteLLM standard)
|
| 95 |
+
{
|
| 96 |
+
"name": "litellm_web_search",
|
| 97 |
+
"description": "Search the web for information...",
|
| 98 |
+
"input_schema": {...}
|
| 99 |
+
}
|
| 100 |
+
```
|
| 101 |
+
|
| 102 |
+
---
|
| 103 |
+
|
| 104 |
+
## Request Flow
|
| 105 |
+
|
| 106 |
+
### Without Interception (Client-Side)
|
| 107 |
+
User manually handles tool execution:
|
| 108 |
+
1. User calls `litellm.messages.acreate()` → Gets `tool_use` response
|
| 109 |
+
2. User executes `litellm.asearch()`
|
| 110 |
+
3. User calls `litellm.messages.acreate()` again with results
|
| 111 |
+
4. User gets final answer
|
| 112 |
+
|
| 113 |
+
**Result**: 2 API calls, manual tool execution
|
| 114 |
+
|
| 115 |
+
### With Interception (Server-Side)
|
| 116 |
+
Server handles tool execution automatically:
|
| 117 |
+
|
| 118 |
+
```mermaid
|
| 119 |
+
sequenceDiagram
|
| 120 |
+
participant User
|
| 121 |
+
participant Messages as litellm.messages.acreate()
|
| 122 |
+
participant Handler as llm_http_handler.py
|
| 123 |
+
participant Logger as WebSearchInterceptionLogger
|
| 124 |
+
participant Router as proxy_server.llm_router
|
| 125 |
+
participant Search as litellm.asearch()
|
| 126 |
+
participant Provider as Bedrock API
|
| 127 |
+
|
| 128 |
+
User->>Messages: acreate(tools=[WebSearch])
|
| 129 |
+
Messages->>Handler: async_anthropic_messages_handler()
|
| 130 |
+
Handler->>Provider: Request
|
| 131 |
+
Provider-->>Handler: Response (tool_use)
|
| 132 |
+
Handler->>Logger: async_should_run_agentic_loop()
|
| 133 |
+
Logger->>Logger: Detect WebSearch tool_use
|
| 134 |
+
Logger-->>Handler: (True, tools)
|
| 135 |
+
Handler->>Logger: async_run_agentic_loop(tools)
|
| 136 |
+
Logger->>Router: Get search_provider from search_tools
|
| 137 |
+
Router-->>Logger: search_provider
|
| 138 |
+
Logger->>Search: asearch(query, provider)
|
| 139 |
+
Search-->>Logger: Search results
|
| 140 |
+
Logger->>Logger: Build tool_result message
|
| 141 |
+
Logger->>Messages: acreate() with results
|
| 142 |
+
Messages->>Provider: Request with search results
|
| 143 |
+
Provider-->>Messages: Final answer
|
| 144 |
+
Messages-->>Logger: Final response
|
| 145 |
+
Logger-->>Handler: Final response
|
| 146 |
+
Handler-->>User: Final answer (with search results)
|
| 147 |
+
```
|
| 148 |
+
|
| 149 |
+
**Result**: 1 API call from user, server handles agentic loop
|
| 150 |
+
|
| 151 |
+
---
|
| 152 |
+
|
| 153 |
+
## Key Components
|
| 154 |
+
|
| 155 |
+
| Component | File | Purpose |
|
| 156 |
+
|-----------|------|---------|
|
| 157 |
+
| **WebSearchInterceptionLogger** | `handler.py` | CustomLogger that implements agentic loop hooks |
|
| 158 |
+
| **Tool Standardization** | `tools.py` | Standard tool definition, detection, and utilities |
|
| 159 |
+
| **Tool Name Constant** | `constants.py` | `LITELLM_WEB_SEARCH_TOOL_NAME = "litellm_web_search"` |
|
| 160 |
+
| **Tool Conversion** | `anthropic/.../ handler.py` | Converts native tools to LiteLLM standard before API call |
|
| 161 |
+
| **Transformation Logic** | `transformation.py` | Detect tool_use, build tool_result messages, format search responses |
|
| 162 |
+
| **Agentic Loop Hooks** | `integrations/custom_logger.py` | Base hooks: `async_should_run_agentic_loop()`, `async_run_agentic_loop()` |
|
| 163 |
+
| **Hook Orchestration** | `llms/custom_httpx/llm_http_handler.py` | `_call_agentic_completion_hooks()` - calls hooks after response |
|
| 164 |
+
| **Router Search Tools** | `proxy/proxy_server.py` | `llm_router.search_tools` - configured search providers |
|
| 165 |
+
| **Search Endpoints** | `proxy/search_endpoints/endpoints.py` | Router logic for selecting search provider |
|
| 166 |
+
|
| 167 |
+
---
|
| 168 |
+
|
| 169 |
+
## Configuration
|
| 170 |
+
|
| 171 |
+
```python
|
| 172 |
+
from litellm.integrations.websearch_interception import (
|
| 173 |
+
WebSearchInterceptionLogger,
|
| 174 |
+
get_litellm_web_search_tool,
|
| 175 |
+
)
|
| 176 |
+
from litellm.types.utils import LlmProviders
|
| 177 |
+
|
| 178 |
+
# Enable for Bedrock with specific search tool
|
| 179 |
+
litellm.callbacks = [
|
| 180 |
+
WebSearchInterceptionLogger(
|
| 181 |
+
enabled_providers=[LlmProviders.BEDROCK],
|
| 182 |
+
search_tool_name="my-perplexity-tool" # Optional: uses router's first tool if None
|
| 183 |
+
)
|
| 184 |
+
]
|
| 185 |
+
|
| 186 |
+
# Make request with LiteLLM standard tool (recommended)
|
| 187 |
+
response = await litellm.messages.acreate(
|
| 188 |
+
model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
| 189 |
+
messages=[{"role": "user", "content": "What is LiteLLM?"}],
|
| 190 |
+
tools=[get_litellm_web_search_tool()], # LiteLLM standard
|
| 191 |
+
max_tokens=1024,
|
| 192 |
+
stream=True # Auto-converted to non-streaming
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
# OR send native tools - they're auto-converted to LiteLLM standard
|
| 196 |
+
response = await litellm.messages.acreate(
|
| 197 |
+
model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
| 198 |
+
messages=[{"role": "user", "content": "What is LiteLLM?"}],
|
| 199 |
+
tools=[{
|
| 200 |
+
"type": "web_search_20250305", # Native Anthropic format
|
| 201 |
+
"name": "web_search",
|
| 202 |
+
"max_uses": 8
|
| 203 |
+
}],
|
| 204 |
+
max_tokens=1024,
|
| 205 |
+
)
|
| 206 |
+
```
|
| 207 |
+
|
| 208 |
+
---
|
| 209 |
+
|
| 210 |
+
## Streaming Support
|
| 211 |
+
|
| 212 |
+
WebSearch interception works transparently with both streaming and non-streaming requests.
|
| 213 |
+
|
| 214 |
+
**How streaming is handled:**
|
| 215 |
+
1. User makes request with `stream=True` and WebSearch tool
|
| 216 |
+
2. Before API call, `anthropic_messages()` detects WebSearch + interception enabled
|
| 217 |
+
3. Converts `stream=True` → `stream=False` internally
|
| 218 |
+
4. Agentic loop executes with non-streaming responses
|
| 219 |
+
5. Final response returned to user (non-streaming)
|
| 220 |
+
|
| 221 |
+
**Why this approach:**
|
| 222 |
+
- Server-side agentic loops require consuming full responses to detect tool_use
|
| 223 |
+
- User opts into this behavior by enabling WebSearch interception
|
| 224 |
+
- Provides seamless experience without client changes
|
| 225 |
+
|
| 226 |
+
**Testing:**
|
| 227 |
+
- **Non-streaming**: `test_websearch_interception_e2e.py`
|
| 228 |
+
- **Streaming**: `test_websearch_interception_streaming_e2e.py`
|
| 229 |
+
|
| 230 |
+
---
|
| 231 |
+
|
| 232 |
+
## Search Provider Selection
|
| 233 |
+
|
| 234 |
+
1. If `search_tool_name` specified → Look up in `llm_router.search_tools`
|
| 235 |
+
2. If not found or None → Use first available search tool
|
| 236 |
+
3. If no router or no tools → Fallback to `perplexity`
|
| 237 |
+
|
| 238 |
+
Example router config:
|
| 239 |
+
```yaml
|
| 240 |
+
search_tools:
|
| 241 |
+
- search_tool_name: "my-perplexity-tool"
|
| 242 |
+
litellm_params:
|
| 243 |
+
search_provider: "perplexity"
|
| 244 |
+
- search_tool_name: "my-tavily-tool"
|
| 245 |
+
litellm_params:
|
| 246 |
+
search_provider: "tavily"
|
| 247 |
+
```
|
| 248 |
+
|
| 249 |
+
---
|
| 250 |
+
|
| 251 |
+
## Message Flow
|
| 252 |
+
|
| 253 |
+
### Initial Request
|
| 254 |
+
```python
|
| 255 |
+
messages = [{"role": "user", "content": "What is LiteLLM?"}]
|
| 256 |
+
tools = [{"name": "WebSearch", ...}]
|
| 257 |
+
```
|
| 258 |
+
|
| 259 |
+
### First API Call (Internal)
|
| 260 |
+
**Response**: `tool_use` with `name="WebSearch"`, `input={"query": "what is litellm"}`
|
| 261 |
+
|
| 262 |
+
### Server Processing
|
| 263 |
+
1. Logger detects WebSearch tool_use
|
| 264 |
+
2. Looks up search provider from router
|
| 265 |
+
3. Executes `litellm.asearch(query="what is litellm", search_provider="perplexity")`
|
| 266 |
+
4. Gets results: `"Title: LiteLLM Docs\nURL: docs.litellm.ai\n..."`
|
| 267 |
+
|
| 268 |
+
### Follow-Up Request (Internal)
|
| 269 |
+
```python
|
| 270 |
+
messages = [
|
| 271 |
+
{"role": "user", "content": "What is LiteLLM?"},
|
| 272 |
+
{"role": "assistant", "content": [{"type": "tool_use", ...}]},
|
| 273 |
+
{"role": "user", "content": [{"type": "tool_result", "content": "search results..."}]}
|
| 274 |
+
]
|
| 275 |
+
```
|
| 276 |
+
|
| 277 |
+
### User Receives
|
| 278 |
+
```python
|
| 279 |
+
response.content[0].text
|
| 280 |
+
# "Based on the search results, LiteLLM is a unified interface..."
|
| 281 |
+
```
|
| 282 |
+
|
| 283 |
+
---
|
| 284 |
+
|
| 285 |
+
## Testing
|
| 286 |
+
|
| 287 |
+
**E2E Tests**:
|
| 288 |
+
- `test_websearch_interception_e2e.py` - Non-streaming real API calls to Bedrock
|
| 289 |
+
- `test_websearch_interception_streaming_e2e.py` - Streaming real API calls to Bedrock
|
| 290 |
+
|
| 291 |
+
**Unit Tests**: `test_websearch_interception.py`
|
| 292 |
+
Mocked tests for tool detection, provider filtering, edge cases.
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/handler.py
ADDED
|
@@ -0,0 +1,1065 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
WebSearch Interception Handler
|
| 3 |
+
|
| 4 |
+
CustomLogger that intercepts WebSearch tool calls for models that don't
|
| 5 |
+
natively support web search (e.g., Bedrock/Claude) and executes them
|
| 6 |
+
server-side using litellm router's search tools.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import asyncio
|
| 10 |
+
import math
|
| 11 |
+
import uuid
|
| 12 |
+
from typing import Any, Dict, List, Optional, Tuple, Union, cast
|
| 13 |
+
|
| 14 |
+
import litellm
|
| 15 |
+
from litellm._logging import verbose_logger
|
| 16 |
+
from litellm.anthropic_interface import messages as anthropic_messages
|
| 17 |
+
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
|
| 18 |
+
from litellm.integrations.custom_logger import CustomLogger
|
| 19 |
+
from litellm.integrations.websearch_interception.tools import (
|
| 20 |
+
get_litellm_web_search_tool,
|
| 21 |
+
get_litellm_web_search_tool_openai,
|
| 22 |
+
is_web_search_tool,
|
| 23 |
+
is_web_search_tool_chat_completion,
|
| 24 |
+
)
|
| 25 |
+
from litellm.integrations.websearch_interception.transformation import (
|
| 26 |
+
WebSearchTransformation,
|
| 27 |
+
)
|
| 28 |
+
from litellm.types.integrations.websearch_interception import (
|
| 29 |
+
WebSearchInterceptionConfig,
|
| 30 |
+
)
|
| 31 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 32 |
+
from litellm.types.utils import LlmProviders
|
| 33 |
+
from litellm.utils import ProviderConfigManager
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class WebSearchInterceptionLogger(CustomLogger):
|
| 37 |
+
"""
|
| 38 |
+
CustomLogger that intercepts WebSearch tool calls for models that don't
|
| 39 |
+
natively support web search.
|
| 40 |
+
|
| 41 |
+
Implements agentic loop:
|
| 42 |
+
1. Detects WebSearch tool_use in model response
|
| 43 |
+
2. Executes litellm.asearch() for each query using router's search tools
|
| 44 |
+
3. Makes follow-up request with search results
|
| 45 |
+
4. Returns final response
|
| 46 |
+
"""
|
| 47 |
+
|
| 48 |
+
def __init__(
|
| 49 |
+
self,
|
| 50 |
+
enabled_providers: Optional[List[Union[LlmProviders, str]]] = None,
|
| 51 |
+
search_tool_name: Optional[str] = None,
|
| 52 |
+
):
|
| 53 |
+
"""
|
| 54 |
+
Args:
|
| 55 |
+
enabled_providers: List of LLM providers to enable interception for.
|
| 56 |
+
Use LlmProviders enum values (e.g., [LlmProviders.BEDROCK])
|
| 57 |
+
If None or empty list, enables for ALL providers.
|
| 58 |
+
Default: None (all providers enabled)
|
| 59 |
+
search_tool_name: Name of search tool configured in router's search_tools.
|
| 60 |
+
If None, will attempt to use first available search tool.
|
| 61 |
+
"""
|
| 62 |
+
super().__init__()
|
| 63 |
+
# Convert enum values to strings for comparison
|
| 64 |
+
if enabled_providers is None:
|
| 65 |
+
self.enabled_providers = [LlmProviders.BEDROCK.value]
|
| 66 |
+
else:
|
| 67 |
+
self.enabled_providers = [
|
| 68 |
+
p.value if isinstance(p, LlmProviders) else p for p in enabled_providers
|
| 69 |
+
]
|
| 70 |
+
self.search_tool_name = search_tool_name
|
| 71 |
+
self._request_has_websearch = False # Track if current request has web search
|
| 72 |
+
|
| 73 |
+
async def try_short_circuit_search(
|
| 74 |
+
self,
|
| 75 |
+
model: str,
|
| 76 |
+
messages: List[Dict],
|
| 77 |
+
tools: Optional[List[Dict]],
|
| 78 |
+
custom_llm_provider: Optional[str],
|
| 79 |
+
) -> Optional[Dict[str, Any]]:
|
| 80 |
+
"""
|
| 81 |
+
Short-circuit web-search-only requests by executing the search directly.
|
| 82 |
+
|
| 83 |
+
Claude Code sends web search as a separate, standalone /v1/messages
|
| 84 |
+
request with a simple prompt and only web_search tool(s). For providers
|
| 85 |
+
that don't natively support web search (e.g. github_copilot), there is
|
| 86 |
+
no need to route this through the backend LLM — we can detect the
|
| 87 |
+
pattern, execute the search via Tavily/Perplexity, and return a
|
| 88 |
+
synthetic Anthropic response immediately.
|
| 89 |
+
|
| 90 |
+
Args:
|
| 91 |
+
model: Model name from the request
|
| 92 |
+
messages: Messages list from the request
|
| 93 |
+
tools: Tools list from the request
|
| 94 |
+
custom_llm_provider: Provider name
|
| 95 |
+
|
| 96 |
+
Returns:
|
| 97 |
+
An AnthropicMessagesResponse dict if short-circuited, or None to
|
| 98 |
+
continue normal processing.
|
| 99 |
+
"""
|
| 100 |
+
if not tools:
|
| 101 |
+
return None
|
| 102 |
+
|
| 103 |
+
# Check if provider is in enabled list
|
| 104 |
+
provider_str = custom_llm_provider or ""
|
| 105 |
+
if (
|
| 106 |
+
self.enabled_providers is not None
|
| 107 |
+
and provider_str not in self.enabled_providers
|
| 108 |
+
):
|
| 109 |
+
return None
|
| 110 |
+
|
| 111 |
+
# Only short-circuit for providers without native Anthropic Messages
|
| 112 |
+
# support. Providers that have a BaseAnthropicMessagesConfig (bedrock,
|
| 113 |
+
# vertex_ai, azure_ai, anthropic) already use the agentic loop, which
|
| 114 |
+
# includes a follow-up LLM call to synthesize the answer from search
|
| 115 |
+
# results. Short-circuiting those would skip that synthesis step and
|
| 116 |
+
# return raw search text — a regression for existing users.
|
| 117 |
+
try:
|
| 118 |
+
provider_enum = LlmProviders(provider_str)
|
| 119 |
+
anthropic_config = (
|
| 120 |
+
ProviderConfigManager.get_provider_anthropic_messages_config(
|
| 121 |
+
model=model, provider=provider_enum
|
| 122 |
+
)
|
| 123 |
+
)
|
| 124 |
+
if anthropic_config is not None:
|
| 125 |
+
verbose_logger.debug(
|
| 126 |
+
f"WebSearchInterception: Skipping short-circuit for {provider_str} "
|
| 127 |
+
"(provider has native Anthropic Messages support, using agentic loop)"
|
| 128 |
+
)
|
| 129 |
+
return None
|
| 130 |
+
except (ValueError, Exception):
|
| 131 |
+
pass # unknown provider enum → safe to short-circuit
|
| 132 |
+
|
| 133 |
+
# All tools must be web search tools
|
| 134 |
+
if not all(is_web_search_tool(t) for t in tools):
|
| 135 |
+
return None
|
| 136 |
+
|
| 137 |
+
# Extract search query from the last user message
|
| 138 |
+
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
| 139 |
+
get_last_user_message,
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
query = get_last_user_message(cast(List[AllMessageValues], messages))
|
| 143 |
+
if not query:
|
| 144 |
+
return None
|
| 145 |
+
|
| 146 |
+
verbose_logger.debug(
|
| 147 |
+
"WebSearchInterception: Short-circuit search detected "
|
| 148 |
+
f"(provider={provider_str}, query='{query}')"
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
# Execute search
|
| 152 |
+
try:
|
| 153 |
+
search_result_text = await self._execute_search(query)
|
| 154 |
+
except Exception as e:
|
| 155 |
+
verbose_logger.error(
|
| 156 |
+
f"WebSearchInterception: Short-circuit search failed: {e}"
|
| 157 |
+
)
|
| 158 |
+
search_result_text = f"Search failed: {e}"
|
| 159 |
+
|
| 160 |
+
# Build synthetic Anthropic response
|
| 161 |
+
response: Dict[str, Any] = {
|
| 162 |
+
"id": f"msg_{str(uuid.uuid4())}",
|
| 163 |
+
"type": "message",
|
| 164 |
+
"role": "assistant",
|
| 165 |
+
"model": model,
|
| 166 |
+
"content": [{"type": "text", "text": search_result_text}],
|
| 167 |
+
"stop_reason": "end_turn",
|
| 168 |
+
"stop_sequence": None,
|
| 169 |
+
"usage": {"input_tokens": 0, "output_tokens": 0},
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
verbose_logger.debug(
|
| 173 |
+
"WebSearchInterception: Short-circuit search completed, "
|
| 174 |
+
f"returning synthetic response ({len(search_result_text)} chars)"
|
| 175 |
+
)
|
| 176 |
+
return response
|
| 177 |
+
|
| 178 |
+
async def async_pre_call_deployment_hook(
|
| 179 |
+
self, kwargs: Dict[str, Any], call_type: Optional[Any]
|
| 180 |
+
) -> Optional[dict]:
|
| 181 |
+
"""
|
| 182 |
+
Pre-call hook to convert native Anthropic web_search tools to regular tools.
|
| 183 |
+
|
| 184 |
+
This prevents Bedrock from trying to execute web search server-side (which fails).
|
| 185 |
+
Instead, we convert it to a regular tool so the model returns tool_use blocks
|
| 186 |
+
that we can intercept and execute ourselves.
|
| 187 |
+
"""
|
| 188 |
+
# Check if this is for an enabled provider
|
| 189 |
+
# Try top-level kwargs first, then nested litellm_params, then derive from model name
|
| 190 |
+
custom_llm_provider = kwargs.get("custom_llm_provider", "") or kwargs.get(
|
| 191 |
+
"litellm_params", {}
|
| 192 |
+
).get("custom_llm_provider", "")
|
| 193 |
+
if not custom_llm_provider:
|
| 194 |
+
try:
|
| 195 |
+
_, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
| 196 |
+
model=kwargs.get("model", "")
|
| 197 |
+
)
|
| 198 |
+
except Exception:
|
| 199 |
+
custom_llm_provider = ""
|
| 200 |
+
if custom_llm_provider not in self.enabled_providers:
|
| 201 |
+
return None
|
| 202 |
+
|
| 203 |
+
# Check if request has tools with native web_search
|
| 204 |
+
tools = kwargs.get("tools")
|
| 205 |
+
if not tools:
|
| 206 |
+
return None
|
| 207 |
+
|
| 208 |
+
# Check if any tool is a web search tool (native or already LiteLLM standard)
|
| 209 |
+
has_websearch = any(is_web_search_tool(t) for t in tools)
|
| 210 |
+
|
| 211 |
+
if not has_websearch:
|
| 212 |
+
return None
|
| 213 |
+
|
| 214 |
+
verbose_logger.debug(
|
| 215 |
+
"WebSearchInterception: Converting native web_search tools to LiteLLM standard"
|
| 216 |
+
)
|
| 217 |
+
|
| 218 |
+
# Convert native/custom web_search tools to LiteLLM standard
|
| 219 |
+
converted_tools = []
|
| 220 |
+
for tool in tools:
|
| 221 |
+
if is_web_search_tool(tool):
|
| 222 |
+
# Convert to LiteLLM standard web search tool
|
| 223 |
+
converted_tool = get_litellm_web_search_tool_openai()
|
| 224 |
+
converted_tools.append(converted_tool)
|
| 225 |
+
verbose_logger.debug(
|
| 226 |
+
f"WebSearchInterception: Converted {tool.get('name', 'unknown')} "
|
| 227 |
+
f"(type={tool.get('type', 'none')}) to {LITELLM_WEB_SEARCH_TOOL_NAME}"
|
| 228 |
+
)
|
| 229 |
+
else:
|
| 230 |
+
# Keep other tools as-is
|
| 231 |
+
converted_tools.append(tool)
|
| 232 |
+
|
| 233 |
+
# Update tools in-place and return full kwargs
|
| 234 |
+
kwargs["tools"] = converted_tools
|
| 235 |
+
return kwargs
|
| 236 |
+
|
| 237 |
+
@classmethod
|
| 238 |
+
def from_config_yaml(
|
| 239 |
+
cls, config: WebSearchInterceptionConfig
|
| 240 |
+
) -> "WebSearchInterceptionLogger":
|
| 241 |
+
"""
|
| 242 |
+
Initialize WebSearchInterceptionLogger from proxy config.yaml parameters.
|
| 243 |
+
|
| 244 |
+
Args:
|
| 245 |
+
config: Configuration dictionary from litellm_settings.websearch_interception_params
|
| 246 |
+
|
| 247 |
+
Returns:
|
| 248 |
+
Configured WebSearchInterceptionLogger instance
|
| 249 |
+
|
| 250 |
+
Example:
|
| 251 |
+
From proxy_config.yaml:
|
| 252 |
+
litellm_settings:
|
| 253 |
+
websearch_interception_params:
|
| 254 |
+
enabled_providers: ["bedrock"]
|
| 255 |
+
search_tool_name: "my-perplexity-search"
|
| 256 |
+
|
| 257 |
+
Usage:
|
| 258 |
+
config = litellm_settings.get("websearch_interception_params", {})
|
| 259 |
+
logger = WebSearchInterceptionLogger.from_config_yaml(config)
|
| 260 |
+
"""
|
| 261 |
+
# Extract parameters from config
|
| 262 |
+
enabled_providers_str = config.get("enabled_providers", None)
|
| 263 |
+
search_tool_name = config.get("search_tool_name", None)
|
| 264 |
+
|
| 265 |
+
# Convert string provider names to LlmProviders enum values
|
| 266 |
+
enabled_providers: Optional[List[Union[LlmProviders, str]]] = None
|
| 267 |
+
if enabled_providers_str is not None:
|
| 268 |
+
enabled_providers = []
|
| 269 |
+
for provider in enabled_providers_str:
|
| 270 |
+
try:
|
| 271 |
+
# Try to convert string to LlmProviders enum
|
| 272 |
+
provider_enum = LlmProviders(provider)
|
| 273 |
+
enabled_providers.append(provider_enum)
|
| 274 |
+
except ValueError:
|
| 275 |
+
# If conversion fails, keep as string
|
| 276 |
+
enabled_providers.append(provider)
|
| 277 |
+
|
| 278 |
+
return cls(
|
| 279 |
+
enabled_providers=enabled_providers,
|
| 280 |
+
search_tool_name=search_tool_name,
|
| 281 |
+
)
|
| 282 |
+
|
| 283 |
+
async def async_pre_request_hook(
|
| 284 |
+
self, model: str, messages: List[Dict], kwargs: Dict
|
| 285 |
+
) -> Optional[Dict]:
|
| 286 |
+
"""
|
| 287 |
+
Pre-request hook to convert native web search tools to LiteLLM standard.
|
| 288 |
+
|
| 289 |
+
This hook is called before the API request is made, allowing us to:
|
| 290 |
+
1. Detect native web search tools (web_search_20250305, etc.)
|
| 291 |
+
2. Convert them to LiteLLM standard format (litellm_web_search)
|
| 292 |
+
3. Convert stream=True to stream=False for interception
|
| 293 |
+
|
| 294 |
+
This prevents providers like Bedrock from trying to execute web search
|
| 295 |
+
natively (which fails), and ensures our agentic loop can intercept tool_use.
|
| 296 |
+
|
| 297 |
+
Returns:
|
| 298 |
+
Modified kwargs dict with converted tools, or None if no modifications needed
|
| 299 |
+
"""
|
| 300 |
+
# Check if this request is for an enabled provider
|
| 301 |
+
custom_llm_provider = kwargs.get("litellm_params", {}).get(
|
| 302 |
+
"custom_llm_provider", ""
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
verbose_logger.debug(
|
| 306 |
+
f"WebSearchInterception: Pre-request hook called"
|
| 307 |
+
f" - custom_llm_provider={custom_llm_provider}"
|
| 308 |
+
f" - enabled_providers={self.enabled_providers or 'ALL'}"
|
| 309 |
+
)
|
| 310 |
+
|
| 311 |
+
if (
|
| 312 |
+
self.enabled_providers is not None
|
| 313 |
+
and custom_llm_provider not in self.enabled_providers
|
| 314 |
+
):
|
| 315 |
+
verbose_logger.debug(
|
| 316 |
+
f"WebSearchInterception: Skipping - provider {custom_llm_provider} not in {self.enabled_providers}"
|
| 317 |
+
)
|
| 318 |
+
return None
|
| 319 |
+
|
| 320 |
+
# Check if request has tools
|
| 321 |
+
tools = kwargs.get("tools")
|
| 322 |
+
if not tools:
|
| 323 |
+
return None
|
| 324 |
+
|
| 325 |
+
# Check if any tool is a web search tool
|
| 326 |
+
has_websearch = any(is_web_search_tool(t) for t in tools)
|
| 327 |
+
if not has_websearch:
|
| 328 |
+
return None
|
| 329 |
+
|
| 330 |
+
verbose_logger.debug(
|
| 331 |
+
f"WebSearchInterception: Pre-request hook triggered for provider={custom_llm_provider}"
|
| 332 |
+
)
|
| 333 |
+
|
| 334 |
+
# Convert native web search tools to LiteLLM standard
|
| 335 |
+
converted_tools = []
|
| 336 |
+
for tool in tools:
|
| 337 |
+
if is_web_search_tool(tool):
|
| 338 |
+
standard_tool = get_litellm_web_search_tool()
|
| 339 |
+
converted_tools.append(standard_tool)
|
| 340 |
+
verbose_logger.debug(
|
| 341 |
+
f"WebSearchInterception: Converted {tool.get('name', 'unknown')} "
|
| 342 |
+
f"(type={tool.get('type', 'none')}) to {LITELLM_WEB_SEARCH_TOOL_NAME}"
|
| 343 |
+
)
|
| 344 |
+
else:
|
| 345 |
+
converted_tools.append(tool)
|
| 346 |
+
|
| 347 |
+
# Update kwargs with converted tools
|
| 348 |
+
kwargs["tools"] = converted_tools
|
| 349 |
+
verbose_logger.debug(
|
| 350 |
+
f"WebSearchInterception: Tools after conversion: {[t.get('name') for t in converted_tools]}"
|
| 351 |
+
)
|
| 352 |
+
|
| 353 |
+
# Convert stream=True to stream=False for WebSearch interception
|
| 354 |
+
if kwargs.get("stream"):
|
| 355 |
+
verbose_logger.debug(
|
| 356 |
+
"WebSearchInterception: Converting stream=True to stream=False"
|
| 357 |
+
)
|
| 358 |
+
kwargs["stream"] = False
|
| 359 |
+
kwargs["_websearch_interception_converted_stream"] = True
|
| 360 |
+
|
| 361 |
+
return kwargs
|
| 362 |
+
|
| 363 |
+
async def async_should_run_agentic_loop(
|
| 364 |
+
self,
|
| 365 |
+
response: Any,
|
| 366 |
+
model: str,
|
| 367 |
+
messages: List[Dict],
|
| 368 |
+
tools: Optional[List[Dict]],
|
| 369 |
+
stream: bool,
|
| 370 |
+
custom_llm_provider: str,
|
| 371 |
+
kwargs: Dict,
|
| 372 |
+
) -> Tuple[bool, Dict]:
|
| 373 |
+
"""
|
| 374 |
+
Check if WebSearch tool interception is needed for Anthropic Messages API.
|
| 375 |
+
|
| 376 |
+
This is the legacy method for Anthropic-style responses.
|
| 377 |
+
For chat completions, use async_should_run_chat_completion_agentic_loop instead.
|
| 378 |
+
"""
|
| 379 |
+
|
| 380 |
+
verbose_logger.debug(
|
| 381 |
+
f"WebSearchInterception: Hook called! provider={custom_llm_provider}, stream={stream}"
|
| 382 |
+
)
|
| 383 |
+
verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}")
|
| 384 |
+
|
| 385 |
+
# Check if provider should be intercepted
|
| 386 |
+
# Note: custom_llm_provider is already normalized by get_llm_provider()
|
| 387 |
+
# (e.g., "bedrock/invoke/..." -> "bedrock")
|
| 388 |
+
if (
|
| 389 |
+
self.enabled_providers is not None
|
| 390 |
+
and custom_llm_provider not in self.enabled_providers
|
| 391 |
+
):
|
| 392 |
+
verbose_logger.debug(
|
| 393 |
+
f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})"
|
| 394 |
+
)
|
| 395 |
+
return False, {}
|
| 396 |
+
|
| 397 |
+
# Check if tools include any web search tool (LiteLLM standard or native)
|
| 398 |
+
has_websearch_tool = any(is_web_search_tool(t) for t in (tools or []))
|
| 399 |
+
if not has_websearch_tool:
|
| 400 |
+
verbose_logger.debug("WebSearchInterception: No web search tool in request")
|
| 401 |
+
return False, {}
|
| 402 |
+
|
| 403 |
+
# Detect WebSearch tool_use in response (Anthropic format)
|
| 404 |
+
should_intercept, tool_calls = WebSearchTransformation.transform_request(
|
| 405 |
+
response=response,
|
| 406 |
+
stream=stream,
|
| 407 |
+
response_format="anthropic",
|
| 408 |
+
)
|
| 409 |
+
|
| 410 |
+
if not should_intercept:
|
| 411 |
+
verbose_logger.debug(
|
| 412 |
+
"WebSearchInterception: No WebSearch tool_use detected in response"
|
| 413 |
+
)
|
| 414 |
+
return False, {}
|
| 415 |
+
|
| 416 |
+
verbose_logger.debug(
|
| 417 |
+
f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop"
|
| 418 |
+
)
|
| 419 |
+
|
| 420 |
+
# Extract thinking blocks from response content.
|
| 421 |
+
# When extended thinking is enabled, the model response includes
|
| 422 |
+
# thinking/redacted_thinking blocks that must be preserved and
|
| 423 |
+
# prepended to the follow-up assistant message.
|
| 424 |
+
thinking_blocks: List[Dict] = []
|
| 425 |
+
if isinstance(response, dict):
|
| 426 |
+
content = response.get("content", [])
|
| 427 |
+
else:
|
| 428 |
+
content = getattr(response, "content", []) or []
|
| 429 |
+
|
| 430 |
+
for block in content:
|
| 431 |
+
if isinstance(block, dict):
|
| 432 |
+
block_type = block.get("type")
|
| 433 |
+
else:
|
| 434 |
+
block_type = getattr(block, "type", None)
|
| 435 |
+
|
| 436 |
+
if block_type in ("thinking", "redacted_thinking"):
|
| 437 |
+
if isinstance(block, dict):
|
| 438 |
+
thinking_blocks.append(block)
|
| 439 |
+
else:
|
| 440 |
+
# Convert object to dict using getattr, matching the
|
| 441 |
+
# pattern in _detect_from_non_streaming_response
|
| 442 |
+
thinking_block_dict: Dict = {"type": block_type}
|
| 443 |
+
if block_type == "thinking":
|
| 444 |
+
thinking_block_dict["thinking"] = getattr(block, "thinking", "")
|
| 445 |
+
thinking_block_dict["signature"] = getattr(
|
| 446 |
+
block, "signature", ""
|
| 447 |
+
)
|
| 448 |
+
else: # redacted_thinking
|
| 449 |
+
thinking_block_dict["data"] = getattr(block, "data", "")
|
| 450 |
+
thinking_blocks.append(thinking_block_dict)
|
| 451 |
+
|
| 452 |
+
if thinking_blocks:
|
| 453 |
+
verbose_logger.debug(
|
| 454 |
+
f"WebSearchInterception: Extracted {len(thinking_blocks)} thinking block(s) from response"
|
| 455 |
+
)
|
| 456 |
+
|
| 457 |
+
# Return tools dict with tool calls and thinking blocks
|
| 458 |
+
tools_dict = {
|
| 459 |
+
"tool_calls": tool_calls,
|
| 460 |
+
"tool_type": "websearch",
|
| 461 |
+
"provider": custom_llm_provider,
|
| 462 |
+
"response_format": "anthropic",
|
| 463 |
+
"thinking_blocks": thinking_blocks,
|
| 464 |
+
}
|
| 465 |
+
return True, tools_dict
|
| 466 |
+
|
| 467 |
+
async def async_should_run_chat_completion_agentic_loop(
|
| 468 |
+
self,
|
| 469 |
+
response: Any,
|
| 470 |
+
model: str,
|
| 471 |
+
messages: List[Dict],
|
| 472 |
+
tools: Optional[List[Dict]],
|
| 473 |
+
stream: bool,
|
| 474 |
+
custom_llm_provider: str,
|
| 475 |
+
kwargs: Dict,
|
| 476 |
+
) -> Tuple[bool, Dict]:
|
| 477 |
+
"""
|
| 478 |
+
Check if WebSearch tool interception is needed for Chat Completions API.
|
| 479 |
+
|
| 480 |
+
Similar to async_should_run_agentic_loop but for OpenAI-style chat completions.
|
| 481 |
+
"""
|
| 482 |
+
|
| 483 |
+
verbose_logger.debug(
|
| 484 |
+
f"WebSearchInterception: Chat completion hook called! provider={custom_llm_provider}, stream={stream}"
|
| 485 |
+
)
|
| 486 |
+
verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}")
|
| 487 |
+
|
| 488 |
+
# Check if provider should be intercepted
|
| 489 |
+
if (
|
| 490 |
+
self.enabled_providers is not None
|
| 491 |
+
and custom_llm_provider not in self.enabled_providers
|
| 492 |
+
):
|
| 493 |
+
verbose_logger.debug(
|
| 494 |
+
f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})"
|
| 495 |
+
)
|
| 496 |
+
return False, {}
|
| 497 |
+
|
| 498 |
+
# Check if tools include any web search tool (strict check for chat completions)
|
| 499 |
+
has_websearch_tool = any(
|
| 500 |
+
is_web_search_tool_chat_completion(t) for t in (tools or [])
|
| 501 |
+
)
|
| 502 |
+
if not has_websearch_tool:
|
| 503 |
+
verbose_logger.debug(
|
| 504 |
+
"WebSearchInterception: No litellm_web_search tool in request"
|
| 505 |
+
)
|
| 506 |
+
return False, {}
|
| 507 |
+
|
| 508 |
+
# Detect WebSearch tool_calls in response (OpenAI format)
|
| 509 |
+
should_intercept, tool_calls = WebSearchTransformation.transform_request(
|
| 510 |
+
response=response,
|
| 511 |
+
stream=stream,
|
| 512 |
+
response_format="openai",
|
| 513 |
+
)
|
| 514 |
+
|
| 515 |
+
if not should_intercept:
|
| 516 |
+
verbose_logger.debug(
|
| 517 |
+
"WebSearchInterception: No WebSearch tool_calls detected in response"
|
| 518 |
+
)
|
| 519 |
+
return False, {}
|
| 520 |
+
|
| 521 |
+
verbose_logger.debug(
|
| 522 |
+
f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop"
|
| 523 |
+
)
|
| 524 |
+
|
| 525 |
+
# Return tools dict with tool calls
|
| 526 |
+
tools_dict = {
|
| 527 |
+
"tool_calls": tool_calls,
|
| 528 |
+
"tool_type": "websearch",
|
| 529 |
+
"provider": custom_llm_provider,
|
| 530 |
+
"response_format": "openai",
|
| 531 |
+
}
|
| 532 |
+
return True, tools_dict
|
| 533 |
+
|
| 534 |
+
async def async_run_agentic_loop(
|
| 535 |
+
self,
|
| 536 |
+
tools: Dict,
|
| 537 |
+
model: str,
|
| 538 |
+
messages: List[Dict],
|
| 539 |
+
response: Any,
|
| 540 |
+
anthropic_messages_provider_config: Any,
|
| 541 |
+
anthropic_messages_optional_request_params: Dict,
|
| 542 |
+
logging_obj: Any,
|
| 543 |
+
stream: bool,
|
| 544 |
+
kwargs: Dict,
|
| 545 |
+
) -> Any:
|
| 546 |
+
"""
|
| 547 |
+
Execute agentic loop with WebSearch execution for Anthropic Messages API.
|
| 548 |
+
|
| 549 |
+
This is the legacy method for Anthropic-style responses.
|
| 550 |
+
"""
|
| 551 |
+
|
| 552 |
+
tool_calls = tools["tool_calls"]
|
| 553 |
+
thinking_blocks = tools.get("thinking_blocks", [])
|
| 554 |
+
|
| 555 |
+
verbose_logger.debug(
|
| 556 |
+
f"WebSearchInterception: Executing agentic loop for {len(tool_calls)} search(es)"
|
| 557 |
+
)
|
| 558 |
+
|
| 559 |
+
return await self._execute_agentic_loop(
|
| 560 |
+
model=model,
|
| 561 |
+
messages=messages,
|
| 562 |
+
tool_calls=tool_calls,
|
| 563 |
+
thinking_blocks=thinking_blocks,
|
| 564 |
+
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
|
| 565 |
+
logging_obj=logging_obj,
|
| 566 |
+
stream=stream,
|
| 567 |
+
kwargs=kwargs,
|
| 568 |
+
)
|
| 569 |
+
|
| 570 |
+
async def async_run_chat_completion_agentic_loop(
|
| 571 |
+
self,
|
| 572 |
+
tools: Dict,
|
| 573 |
+
model: str,
|
| 574 |
+
messages: List[Dict],
|
| 575 |
+
response: Any,
|
| 576 |
+
optional_params: Dict,
|
| 577 |
+
logging_obj: Any,
|
| 578 |
+
stream: bool,
|
| 579 |
+
kwargs: Dict,
|
| 580 |
+
) -> Any:
|
| 581 |
+
"""
|
| 582 |
+
Execute agentic loop with WebSearch execution for Chat Completions API.
|
| 583 |
+
|
| 584 |
+
Similar to async_run_agentic_loop but for OpenAI-style chat completions.
|
| 585 |
+
"""
|
| 586 |
+
|
| 587 |
+
tool_calls = tools["tool_calls"]
|
| 588 |
+
response_format = tools.get("response_format", "openai")
|
| 589 |
+
|
| 590 |
+
verbose_logger.debug(
|
| 591 |
+
f"WebSearchInterception: Executing chat completion agentic loop for {len(tool_calls)} search(es)"
|
| 592 |
+
)
|
| 593 |
+
|
| 594 |
+
return await self._execute_chat_completion_agentic_loop(
|
| 595 |
+
model=model,
|
| 596 |
+
messages=messages,
|
| 597 |
+
tool_calls=tool_calls,
|
| 598 |
+
optional_params=optional_params,
|
| 599 |
+
logging_obj=logging_obj,
|
| 600 |
+
stream=stream,
|
| 601 |
+
kwargs=kwargs,
|
| 602 |
+
response_format=response_format,
|
| 603 |
+
)
|
| 604 |
+
|
| 605 |
+
@staticmethod
|
| 606 |
+
def _resolve_max_tokens(
|
| 607 |
+
optional_params: Dict,
|
| 608 |
+
kwargs: Dict,
|
| 609 |
+
) -> int:
|
| 610 |
+
"""Extract max_tokens and validate against thinking.budget_tokens.
|
| 611 |
+
|
| 612 |
+
Anthropic API requires ``max_tokens > thinking.budget_tokens``.
|
| 613 |
+
If the constraint is violated, auto-adjust to ``budget_tokens + 1024``.
|
| 614 |
+
"""
|
| 615 |
+
max_tokens: int = optional_params.get(
|
| 616 |
+
"max_tokens",
|
| 617 |
+
kwargs.get("max_tokens", 1024),
|
| 618 |
+
)
|
| 619 |
+
thinking_param = optional_params.get("thinking")
|
| 620 |
+
if thinking_param and isinstance(thinking_param, dict):
|
| 621 |
+
budget_tokens = thinking_param.get("budget_tokens")
|
| 622 |
+
if (
|
| 623 |
+
budget_tokens is not None
|
| 624 |
+
and isinstance(budget_tokens, (int, float))
|
| 625 |
+
and math.isfinite(budget_tokens)
|
| 626 |
+
and budget_tokens > 0
|
| 627 |
+
):
|
| 628 |
+
if max_tokens <= budget_tokens:
|
| 629 |
+
adjusted = math.ceil(budget_tokens) + 1024
|
| 630 |
+
verbose_logger.debug(
|
| 631 |
+
"WebSearchInterception: max_tokens=%s <= thinking.budget_tokens=%s, "
|
| 632 |
+
"adjusting to %s to satisfy Anthropic API constraint",
|
| 633 |
+
max_tokens,
|
| 634 |
+
budget_tokens,
|
| 635 |
+
adjusted,
|
| 636 |
+
)
|
| 637 |
+
max_tokens = adjusted
|
| 638 |
+
return max_tokens
|
| 639 |
+
|
| 640 |
+
@staticmethod
|
| 641 |
+
def _prepare_followup_kwargs(kwargs: Dict) -> Dict:
|
| 642 |
+
"""Build kwargs for the follow-up call, excluding internal keys.
|
| 643 |
+
|
| 644 |
+
``litellm_logging_obj`` MUST be excluded so the follow-up call creates
|
| 645 |
+
its own ``Logging`` instance via ``function_setup``. Reusing the
|
| 646 |
+
initial call's logging object triggers the dedup flag
|
| 647 |
+
(``has_logged_async_success``) which silently prevents the initial
|
| 648 |
+
call's spend from being recorded — the root cause of the
|
| 649 |
+
SpendLog / AWS billing mismatch.
|
| 650 |
+
"""
|
| 651 |
+
_internal_keys = {"litellm_logging_obj"}
|
| 652 |
+
return {
|
| 653 |
+
k: v
|
| 654 |
+
for k, v in kwargs.items()
|
| 655 |
+
if not k.startswith("_websearch_interception") and k not in _internal_keys
|
| 656 |
+
}
|
| 657 |
+
|
| 658 |
+
async def _execute_agentic_loop(
|
| 659 |
+
self,
|
| 660 |
+
model: str,
|
| 661 |
+
messages: List[Dict],
|
| 662 |
+
tool_calls: List[Dict],
|
| 663 |
+
thinking_blocks: List[Dict],
|
| 664 |
+
anthropic_messages_optional_request_params: Dict,
|
| 665 |
+
logging_obj: Any,
|
| 666 |
+
stream: bool,
|
| 667 |
+
kwargs: Dict,
|
| 668 |
+
) -> Any:
|
| 669 |
+
"""Execute litellm.search() and make follow-up request"""
|
| 670 |
+
|
| 671 |
+
# Extract search queries from tool_use blocks
|
| 672 |
+
search_tasks = []
|
| 673 |
+
for tool_call in tool_calls:
|
| 674 |
+
query = tool_call["input"].get("query")
|
| 675 |
+
if query:
|
| 676 |
+
verbose_logger.debug(
|
| 677 |
+
f"WebSearchInterception: Queuing search for query='{query}'"
|
| 678 |
+
)
|
| 679 |
+
search_tasks.append(self._execute_search(query))
|
| 680 |
+
else:
|
| 681 |
+
verbose_logger.debug(
|
| 682 |
+
f"WebSearchInterception: Tool call {tool_call['id']} has no query"
|
| 683 |
+
)
|
| 684 |
+
# Add empty result for tools without query
|
| 685 |
+
search_tasks.append(self._create_empty_search_result())
|
| 686 |
+
|
| 687 |
+
# Execute searches in parallel
|
| 688 |
+
verbose_logger.debug(
|
| 689 |
+
f"WebSearchInterception: Executing {len(search_tasks)} search(es) in parallel"
|
| 690 |
+
)
|
| 691 |
+
search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
|
| 692 |
+
|
| 693 |
+
# Handle any exceptions in search results
|
| 694 |
+
final_search_results: List[str] = []
|
| 695 |
+
for i, result in enumerate(search_results):
|
| 696 |
+
if isinstance(result, Exception):
|
| 697 |
+
verbose_logger.error(
|
| 698 |
+
f"WebSearchInterception: Search {i} failed with error: {str(result)}"
|
| 699 |
+
)
|
| 700 |
+
final_search_results.append(f"Search failed: {str(result)}")
|
| 701 |
+
elif isinstance(result, str):
|
| 702 |
+
# Explicitly cast to str for type checker
|
| 703 |
+
final_search_results.append(cast(str, result))
|
| 704 |
+
else:
|
| 705 |
+
# Should never happen, but handle for type safety
|
| 706 |
+
verbose_logger.debug(
|
| 707 |
+
f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
|
| 708 |
+
)
|
| 709 |
+
final_search_results.append(str(result))
|
| 710 |
+
|
| 711 |
+
# Build assistant and user messages using transformation
|
| 712 |
+
assistant_message, user_message = WebSearchTransformation.transform_response(
|
| 713 |
+
tool_calls=tool_calls,
|
| 714 |
+
search_results=final_search_results,
|
| 715 |
+
thinking_blocks=thinking_blocks,
|
| 716 |
+
)
|
| 717 |
+
|
| 718 |
+
# Make follow-up request with search results
|
| 719 |
+
# Type cast: user_message is a Dict for Anthropic format (default response_format)
|
| 720 |
+
follow_up_messages = messages + [assistant_message, cast(Dict, user_message)]
|
| 721 |
+
|
| 722 |
+
verbose_logger.debug(
|
| 723 |
+
"WebSearchInterception: Making follow-up request with search results"
|
| 724 |
+
)
|
| 725 |
+
verbose_logger.debug(
|
| 726 |
+
f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}"
|
| 727 |
+
)
|
| 728 |
+
verbose_logger.debug(
|
| 729 |
+
f"WebSearchInterception: Last message (tool_result): {user_message}"
|
| 730 |
+
)
|
| 731 |
+
|
| 732 |
+
# Correlation context for structured logging
|
| 733 |
+
_call_id = getattr(logging_obj, "litellm_call_id", None) or kwargs.get(
|
| 734 |
+
"litellm_call_id", "unknown"
|
| 735 |
+
)
|
| 736 |
+
|
| 737 |
+
full_model_name = model # safe default before try block
|
| 738 |
+
|
| 739 |
+
# Use anthropic_messages.acreate for follow-up request
|
| 740 |
+
try:
|
| 741 |
+
max_tokens = self._resolve_max_tokens(
|
| 742 |
+
anthropic_messages_optional_request_params, kwargs
|
| 743 |
+
)
|
| 744 |
+
|
| 745 |
+
verbose_logger.debug(
|
| 746 |
+
f"WebSearchInterception: Using max_tokens={max_tokens} for follow-up request"
|
| 747 |
+
)
|
| 748 |
+
|
| 749 |
+
# Create a copy of optional params without max_tokens (since we pass it explicitly)
|
| 750 |
+
optional_params_without_max_tokens = {
|
| 751 |
+
k: v
|
| 752 |
+
for k, v in anthropic_messages_optional_request_params.items()
|
| 753 |
+
if k != "max_tokens"
|
| 754 |
+
}
|
| 755 |
+
|
| 756 |
+
kwargs_for_followup = self._prepare_followup_kwargs(kwargs)
|
| 757 |
+
|
| 758 |
+
# Get model from logging_obj.model_call_details["agentic_loop_params"]
|
| 759 |
+
# This preserves the full model name with provider prefix (e.g., "bedrock/invoke/...")
|
| 760 |
+
if logging_obj is not None:
|
| 761 |
+
agentic_params = logging_obj.model_call_details.get(
|
| 762 |
+
"agentic_loop_params", {}
|
| 763 |
+
)
|
| 764 |
+
full_model_name = agentic_params.get("model", model)
|
| 765 |
+
verbose_logger.debug(
|
| 766 |
+
f"WebSearchInterception: Using model name: {full_model_name}"
|
| 767 |
+
)
|
| 768 |
+
|
| 769 |
+
final_response = await anthropic_messages.acreate(
|
| 770 |
+
max_tokens=max_tokens,
|
| 771 |
+
messages=follow_up_messages,
|
| 772 |
+
model=full_model_name,
|
| 773 |
+
**optional_params_without_max_tokens,
|
| 774 |
+
**kwargs_for_followup,
|
| 775 |
+
)
|
| 776 |
+
verbose_logger.debug(
|
| 777 |
+
f"WebSearchInterception: Follow-up request completed, response type: {type(final_response)}"
|
| 778 |
+
)
|
| 779 |
+
verbose_logger.debug(
|
| 780 |
+
f"WebSearchInterception: Final response: {final_response}"
|
| 781 |
+
)
|
| 782 |
+
return final_response
|
| 783 |
+
except Exception as e:
|
| 784 |
+
verbose_logger.exception(
|
| 785 |
+
"WebSearchInterception: Follow-up request failed "
|
| 786 |
+
"[call_id=%s model=%s messages=%d searches=%d]: %s",
|
| 787 |
+
_call_id,
|
| 788 |
+
full_model_name,
|
| 789 |
+
len(follow_up_messages),
|
| 790 |
+
len(final_search_results),
|
| 791 |
+
str(e),
|
| 792 |
+
)
|
| 793 |
+
raise
|
| 794 |
+
|
| 795 |
+
async def _execute_search(self, query: str) -> str:
|
| 796 |
+
"""Execute a single web search using router's search tools"""
|
| 797 |
+
try:
|
| 798 |
+
# Import router from proxy_server
|
| 799 |
+
try:
|
| 800 |
+
from litellm.proxy.proxy_server import llm_router
|
| 801 |
+
except ImportError:
|
| 802 |
+
verbose_logger.debug(
|
| 803 |
+
"WebSearchInterception: Could not import llm_router from proxy_server, "
|
| 804 |
+
"falling back to direct litellm.asearch() with perplexity"
|
| 805 |
+
)
|
| 806 |
+
llm_router = None
|
| 807 |
+
|
| 808 |
+
# Determine search provider from router's search_tools
|
| 809 |
+
search_provider: Optional[str] = None
|
| 810 |
+
if llm_router is not None and hasattr(llm_router, "search_tools"):
|
| 811 |
+
if self.search_tool_name:
|
| 812 |
+
# Find specific search tool by name
|
| 813 |
+
matching_tools = [
|
| 814 |
+
tool
|
| 815 |
+
for tool in llm_router.search_tools
|
| 816 |
+
if tool.get("search_tool_name") == self.search_tool_name
|
| 817 |
+
]
|
| 818 |
+
if matching_tools:
|
| 819 |
+
search_tool = matching_tools[0]
|
| 820 |
+
search_provider = search_tool.get("litellm_params", {}).get(
|
| 821 |
+
"search_provider"
|
| 822 |
+
)
|
| 823 |
+
verbose_logger.debug(
|
| 824 |
+
f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
|
| 825 |
+
f"with provider '{search_provider}'"
|
| 826 |
+
)
|
| 827 |
+
else:
|
| 828 |
+
verbose_logger.debug(
|
| 829 |
+
f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in router, "
|
| 830 |
+
"falling back to first available or perplexity"
|
| 831 |
+
)
|
| 832 |
+
|
| 833 |
+
# If no specific tool or not found, use first available
|
| 834 |
+
if not search_provider and llm_router.search_tools:
|
| 835 |
+
first_tool = llm_router.search_tools[0]
|
| 836 |
+
search_provider = first_tool.get("litellm_params", {}).get(
|
| 837 |
+
"search_provider"
|
| 838 |
+
)
|
| 839 |
+
verbose_logger.debug(
|
| 840 |
+
f"WebSearchInterception: Using first available search tool with provider '{search_provider}'"
|
| 841 |
+
)
|
| 842 |
+
|
| 843 |
+
# Fallback to perplexity if no router or no search tools configured
|
| 844 |
+
if not search_provider:
|
| 845 |
+
search_provider = "perplexity"
|
| 846 |
+
verbose_logger.debug(
|
| 847 |
+
"WebSearchInterception: No search tools configured in router, "
|
| 848 |
+
f"using default provider '{search_provider}'"
|
| 849 |
+
)
|
| 850 |
+
|
| 851 |
+
verbose_logger.debug(
|
| 852 |
+
f"WebSearchInterception: Executing search for '{query}' using provider '{search_provider}'"
|
| 853 |
+
)
|
| 854 |
+
result = await litellm.asearch(query=query, search_provider=search_provider)
|
| 855 |
+
|
| 856 |
+
# Format using transformation function
|
| 857 |
+
search_result_text = WebSearchTransformation.format_search_response(result)
|
| 858 |
+
|
| 859 |
+
verbose_logger.debug(
|
| 860 |
+
f"WebSearchInterception: Search completed for '{query}', got {len(search_result_text)} chars"
|
| 861 |
+
)
|
| 862 |
+
return search_result_text
|
| 863 |
+
except Exception as e:
|
| 864 |
+
verbose_logger.error(
|
| 865 |
+
f"WebSearchInterception: Search failed for '{query}': {str(e)}"
|
| 866 |
+
)
|
| 867 |
+
raise
|
| 868 |
+
|
| 869 |
+
async def _execute_chat_completion_agentic_loop( # noqa: PLR0915
|
| 870 |
+
self,
|
| 871 |
+
model: str,
|
| 872 |
+
messages: List[Dict],
|
| 873 |
+
tool_calls: List[Dict],
|
| 874 |
+
optional_params: Dict,
|
| 875 |
+
logging_obj: Any,
|
| 876 |
+
stream: bool,
|
| 877 |
+
kwargs: Dict,
|
| 878 |
+
response_format: str = "openai",
|
| 879 |
+
) -> Any:
|
| 880 |
+
"""Execute litellm.search() and make follow-up chat completion request"""
|
| 881 |
+
|
| 882 |
+
# Extract search queries from tool_calls
|
| 883 |
+
search_tasks = []
|
| 884 |
+
for tool_call in tool_calls:
|
| 885 |
+
# Handle both Anthropic-style input and OpenAI-style function.arguments
|
| 886 |
+
query = None
|
| 887 |
+
if "input" in tool_call and isinstance(tool_call["input"], dict):
|
| 888 |
+
query = tool_call["input"].get("query")
|
| 889 |
+
elif "function" in tool_call:
|
| 890 |
+
func = tool_call["function"]
|
| 891 |
+
if isinstance(func, dict):
|
| 892 |
+
args = func.get("arguments", {})
|
| 893 |
+
if isinstance(args, dict):
|
| 894 |
+
query = args.get("query")
|
| 895 |
+
|
| 896 |
+
if query:
|
| 897 |
+
verbose_logger.debug(
|
| 898 |
+
f"WebSearchInterception: Queuing search for query='{query}'"
|
| 899 |
+
)
|
| 900 |
+
search_tasks.append(self._execute_search(query))
|
| 901 |
+
else:
|
| 902 |
+
verbose_logger.debug(
|
| 903 |
+
f"WebSearchInterception: Tool call {tool_call.get('id')} has no query"
|
| 904 |
+
)
|
| 905 |
+
# Add empty result for tools without query
|
| 906 |
+
search_tasks.append(self._create_empty_search_result())
|
| 907 |
+
|
| 908 |
+
# Execute searches in parallel
|
| 909 |
+
verbose_logger.debug(
|
| 910 |
+
f"WebSearchInterception: Executing {len(search_tasks)} search(es) in parallel"
|
| 911 |
+
)
|
| 912 |
+
search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
|
| 913 |
+
|
| 914 |
+
# Handle any exceptions in search results
|
| 915 |
+
final_search_results: List[str] = []
|
| 916 |
+
for i, result in enumerate(search_results):
|
| 917 |
+
if isinstance(result, Exception):
|
| 918 |
+
verbose_logger.error(
|
| 919 |
+
f"WebSearchInterception: Search {i} failed with error: {str(result)}"
|
| 920 |
+
)
|
| 921 |
+
final_search_results.append(f"Search failed: {str(result)}")
|
| 922 |
+
elif isinstance(result, str):
|
| 923 |
+
final_search_results.append(cast(str, result))
|
| 924 |
+
else:
|
| 925 |
+
verbose_logger.debug(
|
| 926 |
+
f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
|
| 927 |
+
)
|
| 928 |
+
final_search_results.append(str(result))
|
| 929 |
+
|
| 930 |
+
# Build assistant and tool messages using transformation
|
| 931 |
+
(
|
| 932 |
+
assistant_message,
|
| 933 |
+
tool_messages_or_user,
|
| 934 |
+
) = WebSearchTransformation.transform_response(
|
| 935 |
+
tool_calls=tool_calls,
|
| 936 |
+
search_results=final_search_results,
|
| 937 |
+
response_format=response_format,
|
| 938 |
+
)
|
| 939 |
+
|
| 940 |
+
# Make follow-up request with search results
|
| 941 |
+
# For OpenAI format, tool_messages_or_user is a list of tool messages
|
| 942 |
+
if response_format == "openai":
|
| 943 |
+
follow_up_messages = (
|
| 944 |
+
messages + [assistant_message] + cast(List[Dict], tool_messages_or_user)
|
| 945 |
+
)
|
| 946 |
+
else:
|
| 947 |
+
# For Anthropic format (shouldn't happen in this method, but handle it)
|
| 948 |
+
follow_up_messages = messages + [
|
| 949 |
+
assistant_message,
|
| 950 |
+
cast(Dict, tool_messages_or_user),
|
| 951 |
+
]
|
| 952 |
+
|
| 953 |
+
verbose_logger.debug(
|
| 954 |
+
"WebSearchInterception: Making follow-up chat completion request with search results"
|
| 955 |
+
)
|
| 956 |
+
verbose_logger.debug(
|
| 957 |
+
f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}"
|
| 958 |
+
)
|
| 959 |
+
|
| 960 |
+
# Use litellm.acompletion for follow-up request
|
| 961 |
+
try:
|
| 962 |
+
# Remove internal parameters that shouldn't be passed to follow-up request
|
| 963 |
+
internal_params = {
|
| 964 |
+
"_websearch_interception",
|
| 965 |
+
"acompletion",
|
| 966 |
+
"litellm_logging_obj",
|
| 967 |
+
"custom_llm_provider",
|
| 968 |
+
"model_alias_map",
|
| 969 |
+
"stream_response",
|
| 970 |
+
"custom_prompt_dict",
|
| 971 |
+
}
|
| 972 |
+
kwargs_for_followup = {
|
| 973 |
+
k: v
|
| 974 |
+
for k, v in kwargs.items()
|
| 975 |
+
if not k.startswith("_websearch_interception")
|
| 976 |
+
and k not in internal_params
|
| 977 |
+
}
|
| 978 |
+
|
| 979 |
+
# Get full model name from kwargs
|
| 980 |
+
full_model_name = model
|
| 981 |
+
if "custom_llm_provider" in kwargs:
|
| 982 |
+
custom_llm_provider = kwargs["custom_llm_provider"]
|
| 983 |
+
# Reconstruct full model name with provider prefix if needed
|
| 984 |
+
if not model.startswith(custom_llm_provider):
|
| 985 |
+
# Check if model already has a provider prefix
|
| 986 |
+
if "/" not in model:
|
| 987 |
+
full_model_name = f"{custom_llm_provider}/{model}"
|
| 988 |
+
|
| 989 |
+
verbose_logger.debug(
|
| 990 |
+
f"WebSearchInterception: Using model name: {full_model_name}"
|
| 991 |
+
)
|
| 992 |
+
|
| 993 |
+
# Prepare tools for follow-up request (same as original)
|
| 994 |
+
tools_param = optional_params.get("tools")
|
| 995 |
+
|
| 996 |
+
# Remove tools and extra_body from optional_params to avoid issues
|
| 997 |
+
# extra_body often contains internal LiteLLM params that shouldn't be forwarded
|
| 998 |
+
optional_params_clean = {
|
| 999 |
+
k: v
|
| 1000 |
+
for k, v in optional_params.items()
|
| 1001 |
+
if k
|
| 1002 |
+
not in {
|
| 1003 |
+
"tools",
|
| 1004 |
+
"extra_body",
|
| 1005 |
+
"model_alias_map",
|
| 1006 |
+
"stream_response",
|
| 1007 |
+
"custom_prompt_dict",
|
| 1008 |
+
}
|
| 1009 |
+
}
|
| 1010 |
+
|
| 1011 |
+
final_response = await litellm.acompletion(
|
| 1012 |
+
model=full_model_name,
|
| 1013 |
+
messages=follow_up_messages,
|
| 1014 |
+
tools=tools_param,
|
| 1015 |
+
**optional_params_clean,
|
| 1016 |
+
**kwargs_for_followup,
|
| 1017 |
+
)
|
| 1018 |
+
|
| 1019 |
+
verbose_logger.debug(
|
| 1020 |
+
f"WebSearchInterception: Follow-up request completed, response type: {type(final_response)}"
|
| 1021 |
+
)
|
| 1022 |
+
return final_response
|
| 1023 |
+
except Exception as e:
|
| 1024 |
+
verbose_logger.exception(
|
| 1025 |
+
f"WebSearchInterception: Follow-up request failed: {str(e)}"
|
| 1026 |
+
)
|
| 1027 |
+
raise
|
| 1028 |
+
|
| 1029 |
+
async def _create_empty_search_result(self) -> str:
|
| 1030 |
+
"""Create an empty search result for tool calls without queries"""
|
| 1031 |
+
return "No search query provided"
|
| 1032 |
+
|
| 1033 |
+
@staticmethod
|
| 1034 |
+
def initialize_from_proxy_config(
|
| 1035 |
+
litellm_settings: Dict[str, Any],
|
| 1036 |
+
callback_specific_params: Dict[str, Any],
|
| 1037 |
+
) -> "WebSearchInterceptionLogger":
|
| 1038 |
+
"""
|
| 1039 |
+
Static method to initialize WebSearchInterceptionLogger from proxy config.
|
| 1040 |
+
|
| 1041 |
+
Used in callback_utils.py to simplify initialization logic.
|
| 1042 |
+
|
| 1043 |
+
Args:
|
| 1044 |
+
litellm_settings: Dictionary containing litellm_settings from proxy_config.yaml
|
| 1045 |
+
callback_specific_params: Dictionary containing callback-specific parameters
|
| 1046 |
+
|
| 1047 |
+
Returns:
|
| 1048 |
+
Configured WebSearchInterceptionLogger instance
|
| 1049 |
+
|
| 1050 |
+
Example:
|
| 1051 |
+
From callback_utils.py:
|
| 1052 |
+
websearch_obj = WebSearchInterceptionLogger.initialize_from_proxy_config(
|
| 1053 |
+
litellm_settings=litellm_settings,
|
| 1054 |
+
callback_specific_params=callback_specific_params
|
| 1055 |
+
)
|
| 1056 |
+
"""
|
| 1057 |
+
# Get websearch_interception_params from litellm_settings or callback_specific_params
|
| 1058 |
+
websearch_params: WebSearchInterceptionConfig = {}
|
| 1059 |
+
if "websearch_interception_params" in litellm_settings:
|
| 1060 |
+
websearch_params = litellm_settings["websearch_interception_params"]
|
| 1061 |
+
elif "websearch_interception" in callback_specific_params:
|
| 1062 |
+
websearch_params = callback_specific_params["websearch_interception"]
|
| 1063 |
+
|
| 1064 |
+
# Use classmethod to initialize from config
|
| 1065 |
+
return WebSearchInterceptionLogger.from_config_yaml(websearch_params)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/tools.py
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
LiteLLM Web Search Tool Definition
|
| 3 |
+
|
| 4 |
+
This module defines the standard web search tool used across LiteLLM.
|
| 5 |
+
Native provider tools (like Anthropic's web_search_20250305) are converted
|
| 6 |
+
to this format for consistent interception and execution.
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from typing import Any, Dict
|
| 10 |
+
|
| 11 |
+
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def get_litellm_web_search_tool() -> Dict[str, Any]:
|
| 15 |
+
"""
|
| 16 |
+
Get the standard LiteLLM web search tool definition.
|
| 17 |
+
|
| 18 |
+
This is the canonical tool definition that all native web search tools
|
| 19 |
+
(like Anthropic's web_search_20250305, Claude Code's web_search, etc.)
|
| 20 |
+
are converted to for interception.
|
| 21 |
+
|
| 22 |
+
Returns:
|
| 23 |
+
Dict containing the Anthropic-style tool definition with:
|
| 24 |
+
- name: Tool name
|
| 25 |
+
- description: What the tool does
|
| 26 |
+
- input_schema: JSON schema for tool parameters
|
| 27 |
+
|
| 28 |
+
Example:
|
| 29 |
+
>>> tool = get_litellm_web_search_tool()
|
| 30 |
+
>>> tool['name']
|
| 31 |
+
'litellm_web_search'
|
| 32 |
+
"""
|
| 33 |
+
return {
|
| 34 |
+
"name": LITELLM_WEB_SEARCH_TOOL_NAME,
|
| 35 |
+
"description": (
|
| 36 |
+
"Search the web for information. Use this when you need current "
|
| 37 |
+
"information or answers to questions that require up-to-date data."
|
| 38 |
+
),
|
| 39 |
+
"input_schema": {
|
| 40 |
+
"type": "object",
|
| 41 |
+
"properties": {
|
| 42 |
+
"query": {
|
| 43 |
+
"type": "string",
|
| 44 |
+
"description": "The search query to execute",
|
| 45 |
+
}
|
| 46 |
+
},
|
| 47 |
+
"required": ["query"],
|
| 48 |
+
},
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def get_litellm_web_search_tool_openai() -> Dict[str, Any]:
|
| 53 |
+
"""
|
| 54 |
+
Get the standard LiteLLM web search tool definition in OpenAI format.
|
| 55 |
+
|
| 56 |
+
Used by async_pre_call_deployment_hook which runs in the chat completions
|
| 57 |
+
path where tools must be in OpenAI format (type: "function" with
|
| 58 |
+
function.parameters).
|
| 59 |
+
|
| 60 |
+
Returns:
|
| 61 |
+
Dict containing the OpenAI-style tool definition.
|
| 62 |
+
"""
|
| 63 |
+
return {
|
| 64 |
+
"type": "function",
|
| 65 |
+
"function": {
|
| 66 |
+
"name": LITELLM_WEB_SEARCH_TOOL_NAME,
|
| 67 |
+
"description": (
|
| 68 |
+
"Search the web for information. Use this when you need current "
|
| 69 |
+
"information or answers to questions that require up-to-date data."
|
| 70 |
+
),
|
| 71 |
+
"parameters": {
|
| 72 |
+
"type": "object",
|
| 73 |
+
"properties": {
|
| 74 |
+
"query": {
|
| 75 |
+
"type": "string",
|
| 76 |
+
"description": "The search query to execute",
|
| 77 |
+
}
|
| 78 |
+
},
|
| 79 |
+
"required": ["query"],
|
| 80 |
+
},
|
| 81 |
+
},
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def is_web_search_tool_chat_completion(tool: Dict[str, Any]) -> bool:
|
| 86 |
+
"""
|
| 87 |
+
Check if a tool is a web search tool for Chat Completions API (strict check).
|
| 88 |
+
|
| 89 |
+
This is a stricter version that ONLY checks for the exact LiteLLM web search tool name.
|
| 90 |
+
Use this for Chat Completions API to avoid false positives with user-defined tools.
|
| 91 |
+
|
| 92 |
+
Detects ONLY:
|
| 93 |
+
- LiteLLM standard: name == "litellm_web_search" (Anthropic format)
|
| 94 |
+
- OpenAI format: type == "function" with function.name == "litellm_web_search"
|
| 95 |
+
|
| 96 |
+
Args:
|
| 97 |
+
tool: Tool dictionary to check
|
| 98 |
+
|
| 99 |
+
Returns:
|
| 100 |
+
True if tool is exactly the LiteLLM web search tool
|
| 101 |
+
|
| 102 |
+
Example:
|
| 103 |
+
>>> is_web_search_tool_chat_completion({"name": "litellm_web_search"})
|
| 104 |
+
True
|
| 105 |
+
>>> is_web_search_tool_chat_completion({"type": "function", "function": {"name": "litellm_web_search"}})
|
| 106 |
+
True
|
| 107 |
+
>>> is_web_search_tool_chat_completion({"name": "web_search"})
|
| 108 |
+
False
|
| 109 |
+
>>> is_web_search_tool_chat_completion({"name": "WebSearch"})
|
| 110 |
+
False
|
| 111 |
+
"""
|
| 112 |
+
tool_name = tool.get("name", "")
|
| 113 |
+
tool_type = tool.get("type", "")
|
| 114 |
+
|
| 115 |
+
# Check for OpenAI format: {"type": "function", "function": {"name": "litellm_web_search"}}
|
| 116 |
+
if tool_type == "function" and "function" in tool:
|
| 117 |
+
function_def = tool.get("function", {})
|
| 118 |
+
function_name = function_def.get("name", "")
|
| 119 |
+
if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
|
| 120 |
+
return True
|
| 121 |
+
|
| 122 |
+
# Check for LiteLLM standard tool (Anthropic format)
|
| 123 |
+
if tool_name == LITELLM_WEB_SEARCH_TOOL_NAME:
|
| 124 |
+
return True
|
| 125 |
+
|
| 126 |
+
return False
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def is_web_search_tool(tool: Dict[str, Any]) -> bool:
|
| 130 |
+
"""
|
| 131 |
+
Check if a tool is a web search tool (native or LiteLLM standard).
|
| 132 |
+
|
| 133 |
+
Detects:
|
| 134 |
+
- LiteLLM standard: name == "litellm_web_search"
|
| 135 |
+
- OpenAI format: type == "function" with function.name == "litellm_web_search"
|
| 136 |
+
- Anthropic native: type starts with "web_search_" (e.g., "web_search_20250305")
|
| 137 |
+
- Claude Code: name == "web_search" with a type field
|
| 138 |
+
- Custom: name == "WebSearch" (legacy format)
|
| 139 |
+
|
| 140 |
+
Args:
|
| 141 |
+
tool: Tool dictionary to check
|
| 142 |
+
|
| 143 |
+
Returns:
|
| 144 |
+
True if tool is a web search tool
|
| 145 |
+
|
| 146 |
+
Example:
|
| 147 |
+
>>> is_web_search_tool({"name": "litellm_web_search"})
|
| 148 |
+
True
|
| 149 |
+
>>> is_web_search_tool({"type": "function", "function": {"name": "litellm_web_search"}})
|
| 150 |
+
True
|
| 151 |
+
>>> is_web_search_tool({"type": "web_search_20250305", "name": "web_search"})
|
| 152 |
+
True
|
| 153 |
+
>>> is_web_search_tool({"name": "calculator"})
|
| 154 |
+
False
|
| 155 |
+
"""
|
| 156 |
+
tool_name = tool.get("name", "")
|
| 157 |
+
tool_type = tool.get("type", "")
|
| 158 |
+
|
| 159 |
+
# Check for OpenAI format: {"type": "function", "function": {"name": "..."}}
|
| 160 |
+
if tool_type == "function" and "function" in tool:
|
| 161 |
+
function_def = tool.get("function", {})
|
| 162 |
+
function_name = function_def.get("name", "")
|
| 163 |
+
if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
|
| 164 |
+
return True
|
| 165 |
+
|
| 166 |
+
# Check for LiteLLM standard tool (Anthropic format)
|
| 167 |
+
if tool_name == LITELLM_WEB_SEARCH_TOOL_NAME:
|
| 168 |
+
return True
|
| 169 |
+
|
| 170 |
+
# Check for native Anthropic web_search_* types
|
| 171 |
+
if tool_type.startswith("web_search_"):
|
| 172 |
+
return True
|
| 173 |
+
|
| 174 |
+
# Check for Claude Code's web_search with a type field
|
| 175 |
+
if tool_name == "web_search" and tool_type:
|
| 176 |
+
return True
|
| 177 |
+
|
| 178 |
+
# Check for legacy WebSearch format
|
| 179 |
+
if tool_name == "WebSearch":
|
| 180 |
+
return True
|
| 181 |
+
|
| 182 |
+
return False
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/transformation.py
ADDED
|
@@ -0,0 +1,373 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
WebSearch Tool Transformation
|
| 3 |
+
|
| 4 |
+
Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format.
|
| 5 |
+
"""
|
| 6 |
+
import json
|
| 7 |
+
from typing import Any, Dict, List, Optional, Tuple, Union
|
| 8 |
+
|
| 9 |
+
from litellm._logging import verbose_logger
|
| 10 |
+
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
|
| 11 |
+
from litellm.llms.base_llm.search.transformation import SearchResponse
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class WebSearchTransformation:
|
| 15 |
+
"""
|
| 16 |
+
Transformation class for WebSearch tool interception.
|
| 17 |
+
|
| 18 |
+
Handles transformation between:
|
| 19 |
+
- Anthropic tool_use format → LiteLLM search requests
|
| 20 |
+
- OpenAI tool_calls format → LiteLLM search requests
|
| 21 |
+
- LiteLLM SearchResponse → Anthropic/OpenAI tool_result format
|
| 22 |
+
"""
|
| 23 |
+
|
| 24 |
+
@staticmethod
|
| 25 |
+
def transform_request(
|
| 26 |
+
response: Any,
|
| 27 |
+
stream: bool,
|
| 28 |
+
response_format: str = "anthropic",
|
| 29 |
+
) -> Tuple[bool, List[Dict]]:
|
| 30 |
+
"""
|
| 31 |
+
Transform model response to extract WebSearch tool calls.
|
| 32 |
+
|
| 33 |
+
Detects if response contains WebSearch tool_use/tool_calls blocks and extracts
|
| 34 |
+
the search queries for execution.
|
| 35 |
+
|
| 36 |
+
Args:
|
| 37 |
+
response: Model response (dict, AnthropicMessagesResponse, or ModelResponse)
|
| 38 |
+
stream: Whether response is streaming
|
| 39 |
+
response_format: Response format - "anthropic" or "openai" (default: "anthropic")
|
| 40 |
+
|
| 41 |
+
Returns:
|
| 42 |
+
(has_websearch, tool_calls):
|
| 43 |
+
has_websearch: True if WebSearch tool_use found
|
| 44 |
+
tool_calls: List of tool_use/tool_calls dicts with id, name, input/function
|
| 45 |
+
|
| 46 |
+
Note:
|
| 47 |
+
Streaming requests are handled by converting stream=True to stream=False
|
| 48 |
+
in the WebSearchInterceptionLogger.async_log_pre_api_call hook before
|
| 49 |
+
the API request is made. This means by the time this method is called,
|
| 50 |
+
streaming requests have already been converted to non-streaming.
|
| 51 |
+
"""
|
| 52 |
+
if stream:
|
| 53 |
+
# This should not happen in practice since we convert streaming to non-streaming
|
| 54 |
+
# in async_log_pre_api_call, but keep this check for safety
|
| 55 |
+
verbose_logger.warning(
|
| 56 |
+
"WebSearchInterception: Unexpected streaming response, skipping interception"
|
| 57 |
+
)
|
| 58 |
+
return False, []
|
| 59 |
+
|
| 60 |
+
# Parse non-streaming response based on format
|
| 61 |
+
if response_format == "openai":
|
| 62 |
+
return WebSearchTransformation._detect_from_openai_response(response)
|
| 63 |
+
else:
|
| 64 |
+
return WebSearchTransformation._detect_from_non_streaming_response(response)
|
| 65 |
+
|
| 66 |
+
@staticmethod
|
| 67 |
+
def _detect_from_non_streaming_response(
|
| 68 |
+
response: Any,
|
| 69 |
+
) -> Tuple[bool, List[Dict]]:
|
| 70 |
+
"""Parse non-streaming response for WebSearch tool_use"""
|
| 71 |
+
|
| 72 |
+
# Handle both dict and object responses
|
| 73 |
+
if isinstance(response, dict):
|
| 74 |
+
content = response.get("content", [])
|
| 75 |
+
else:
|
| 76 |
+
if not hasattr(response, "content"):
|
| 77 |
+
verbose_logger.debug(
|
| 78 |
+
"WebSearchInterception: Response has no content attribute"
|
| 79 |
+
)
|
| 80 |
+
return False, []
|
| 81 |
+
content = response.content or []
|
| 82 |
+
|
| 83 |
+
if not content:
|
| 84 |
+
verbose_logger.debug("WebSearchInterception: Response has empty content")
|
| 85 |
+
return False, []
|
| 86 |
+
|
| 87 |
+
# Find all WebSearch tool_use blocks
|
| 88 |
+
tool_calls = []
|
| 89 |
+
for block in content:
|
| 90 |
+
# Handle both dict and object blocks
|
| 91 |
+
if isinstance(block, dict):
|
| 92 |
+
block_type = block.get("type")
|
| 93 |
+
block_name = block.get("name")
|
| 94 |
+
block_id = block.get("id")
|
| 95 |
+
block_input = block.get("input", {})
|
| 96 |
+
else:
|
| 97 |
+
block_type = getattr(block, "type", None)
|
| 98 |
+
block_name = getattr(block, "name", None)
|
| 99 |
+
block_id = getattr(block, "id", None)
|
| 100 |
+
block_input = getattr(block, "input", {})
|
| 101 |
+
|
| 102 |
+
# Check for LiteLLM standard or legacy web search tools
|
| 103 |
+
# Handles: litellm_web_search, WebSearch, web_search
|
| 104 |
+
if block_type == "tool_use" and block_name in (
|
| 105 |
+
LITELLM_WEB_SEARCH_TOOL_NAME,
|
| 106 |
+
"WebSearch",
|
| 107 |
+
"web_search",
|
| 108 |
+
):
|
| 109 |
+
# Convert to dict for easier handling
|
| 110 |
+
tool_call = {
|
| 111 |
+
"id": block_id,
|
| 112 |
+
"type": "tool_use",
|
| 113 |
+
"name": block_name, # Preserve original name
|
| 114 |
+
"input": block_input,
|
| 115 |
+
}
|
| 116 |
+
tool_calls.append(tool_call)
|
| 117 |
+
verbose_logger.debug(
|
| 118 |
+
f"WebSearchInterception: Found {block_name} tool_use with id={tool_call['id']}"
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
return len(tool_calls) > 0, tool_calls
|
| 122 |
+
|
| 123 |
+
@staticmethod
|
| 124 |
+
def _detect_from_openai_response(
|
| 125 |
+
response: Any,
|
| 126 |
+
) -> Tuple[bool, List[Dict]]:
|
| 127 |
+
"""Parse OpenAI-style response for WebSearch tool_calls"""
|
| 128 |
+
|
| 129 |
+
# Handle both dict and ModelResponse objects
|
| 130 |
+
if isinstance(response, dict):
|
| 131 |
+
choices = response.get("choices", [])
|
| 132 |
+
else:
|
| 133 |
+
if not hasattr(response, "choices"):
|
| 134 |
+
verbose_logger.debug(
|
| 135 |
+
"WebSearchInterception: Response has no choices attribute"
|
| 136 |
+
)
|
| 137 |
+
return False, []
|
| 138 |
+
choices = response.choices or []
|
| 139 |
+
|
| 140 |
+
if not choices:
|
| 141 |
+
verbose_logger.debug("WebSearchInterception: Response has empty choices")
|
| 142 |
+
return False, []
|
| 143 |
+
|
| 144 |
+
# Get first choice's message
|
| 145 |
+
first_choice = choices[0]
|
| 146 |
+
if isinstance(first_choice, dict):
|
| 147 |
+
message = first_choice.get("message", {})
|
| 148 |
+
else:
|
| 149 |
+
message = getattr(first_choice, "message", None)
|
| 150 |
+
|
| 151 |
+
if not message:
|
| 152 |
+
verbose_logger.debug("WebSearchInterception: First choice has no message")
|
| 153 |
+
return False, []
|
| 154 |
+
|
| 155 |
+
# Get tool_calls from message
|
| 156 |
+
if isinstance(message, dict):
|
| 157 |
+
openai_tool_calls = message.get("tool_calls", [])
|
| 158 |
+
else:
|
| 159 |
+
openai_tool_calls = getattr(message, "tool_calls", None) or []
|
| 160 |
+
|
| 161 |
+
if not openai_tool_calls:
|
| 162 |
+
verbose_logger.debug("WebSearchInterception: Message has no tool_calls")
|
| 163 |
+
return False, []
|
| 164 |
+
|
| 165 |
+
# Find all WebSearch tool calls
|
| 166 |
+
tool_calls = []
|
| 167 |
+
for tool_call in openai_tool_calls:
|
| 168 |
+
# Handle both dict and object tool calls
|
| 169 |
+
if isinstance(tool_call, dict):
|
| 170 |
+
tool_id = tool_call.get("id")
|
| 171 |
+
tool_type = tool_call.get("type")
|
| 172 |
+
function = tool_call.get("function", {})
|
| 173 |
+
function_name = (
|
| 174 |
+
function.get("name")
|
| 175 |
+
if isinstance(function, dict)
|
| 176 |
+
else getattr(function, "name", None)
|
| 177 |
+
)
|
| 178 |
+
function_arguments = (
|
| 179 |
+
function.get("arguments")
|
| 180 |
+
if isinstance(function, dict)
|
| 181 |
+
else getattr(function, "arguments", None)
|
| 182 |
+
)
|
| 183 |
+
else:
|
| 184 |
+
tool_id = getattr(tool_call, "id", None)
|
| 185 |
+
tool_type = getattr(tool_call, "type", None)
|
| 186 |
+
function = getattr(tool_call, "function", None)
|
| 187 |
+
function_name = getattr(function, "name", None) if function else None
|
| 188 |
+
function_arguments = (
|
| 189 |
+
getattr(function, "arguments", None) if function else None
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
# Check for LiteLLM standard or legacy web search tools
|
| 193 |
+
if tool_type == "function" and function_name in (
|
| 194 |
+
LITELLM_WEB_SEARCH_TOOL_NAME,
|
| 195 |
+
"WebSearch",
|
| 196 |
+
"web_search",
|
| 197 |
+
):
|
| 198 |
+
# Parse arguments (might be JSON string)
|
| 199 |
+
if isinstance(function_arguments, str):
|
| 200 |
+
try:
|
| 201 |
+
arguments = json.loads(function_arguments)
|
| 202 |
+
except json.JSONDecodeError:
|
| 203 |
+
verbose_logger.warning(
|
| 204 |
+
f"WebSearchInterception: Failed to parse function arguments: {function_arguments}"
|
| 205 |
+
)
|
| 206 |
+
arguments = {}
|
| 207 |
+
else:
|
| 208 |
+
arguments = function_arguments or {}
|
| 209 |
+
|
| 210 |
+
# Convert to internal format (similar to Anthropic)
|
| 211 |
+
tool_call_dict = {
|
| 212 |
+
"id": tool_id,
|
| 213 |
+
"type": "function",
|
| 214 |
+
"name": function_name,
|
| 215 |
+
"function": {
|
| 216 |
+
"name": function_name,
|
| 217 |
+
"arguments": arguments,
|
| 218 |
+
},
|
| 219 |
+
"input": arguments, # For compatibility with Anthropic format
|
| 220 |
+
}
|
| 221 |
+
tool_calls.append(tool_call_dict)
|
| 222 |
+
verbose_logger.debug(
|
| 223 |
+
f"WebSearchInterception: Found {function_name} tool_call with id={tool_id}"
|
| 224 |
+
)
|
| 225 |
+
|
| 226 |
+
return len(tool_calls) > 0, tool_calls
|
| 227 |
+
|
| 228 |
+
@staticmethod
|
| 229 |
+
def transform_response(
|
| 230 |
+
tool_calls: List[Dict],
|
| 231 |
+
search_results: List[str],
|
| 232 |
+
response_format: str = "anthropic",
|
| 233 |
+
thinking_blocks: Optional[List[Dict]] = None,
|
| 234 |
+
) -> Tuple[Dict, Union[Dict, List[Dict]]]:
|
| 235 |
+
"""
|
| 236 |
+
Transform LiteLLM search results to Anthropic/OpenAI tool_result format.
|
| 237 |
+
|
| 238 |
+
Builds the assistant and user/tool messages needed for the agentic loop
|
| 239 |
+
follow-up request.
|
| 240 |
+
|
| 241 |
+
Args:
|
| 242 |
+
tool_calls: List of tool_use/tool_calls dicts from transform_request
|
| 243 |
+
search_results: List of search result strings (one per tool_call)
|
| 244 |
+
response_format: Response format - "anthropic" or "openai" (default: "anthropic")
|
| 245 |
+
thinking_blocks: Optional list of thinking/redacted_thinking blocks
|
| 246 |
+
from the model's response. When present, prepended to the
|
| 247 |
+
assistant message content (required by Anthropic API when
|
| 248 |
+
thinking is enabled).
|
| 249 |
+
|
| 250 |
+
Returns:
|
| 251 |
+
(assistant_message, user_or_tool_messages):
|
| 252 |
+
For Anthropic: assistant_message with tool_use blocks, user_message with tool_result blocks
|
| 253 |
+
For OpenAI: assistant_message with tool_calls, tool_messages list with tool results
|
| 254 |
+
"""
|
| 255 |
+
if response_format == "openai":
|
| 256 |
+
return WebSearchTransformation._transform_response_openai(
|
| 257 |
+
tool_calls, search_results
|
| 258 |
+
)
|
| 259 |
+
else:
|
| 260 |
+
return WebSearchTransformation._transform_response_anthropic(
|
| 261 |
+
tool_calls, search_results, thinking_blocks=thinking_blocks
|
| 262 |
+
)
|
| 263 |
+
|
| 264 |
+
@staticmethod
|
| 265 |
+
def _transform_response_anthropic(
|
| 266 |
+
tool_calls: List[Dict],
|
| 267 |
+
search_results: List[str],
|
| 268 |
+
thinking_blocks: Optional[List[Dict]] = None,
|
| 269 |
+
) -> Tuple[Dict, Dict]:
|
| 270 |
+
"""Transform to Anthropic format (single user message with tool_result blocks)"""
|
| 271 |
+
# Build assistant message content
|
| 272 |
+
assistant_content: List[Dict] = []
|
| 273 |
+
|
| 274 |
+
# Prepend thinking blocks if present.
|
| 275 |
+
# When extended thinking is enabled, Anthropic requires the assistant
|
| 276 |
+
# message to start with thinking/redacted_thinking blocks before any
|
| 277 |
+
# tool_use blocks. Same pattern as anthropic_messages_pt in factory.py.
|
| 278 |
+
if thinking_blocks:
|
| 279 |
+
assistant_content.extend(thinking_blocks)
|
| 280 |
+
|
| 281 |
+
# Add tool_use blocks
|
| 282 |
+
assistant_content.extend(
|
| 283 |
+
[
|
| 284 |
+
{
|
| 285 |
+
"type": "tool_use",
|
| 286 |
+
"id": tc["id"],
|
| 287 |
+
"name": tc["name"],
|
| 288 |
+
"input": tc["input"],
|
| 289 |
+
}
|
| 290 |
+
for tc in tool_calls
|
| 291 |
+
]
|
| 292 |
+
)
|
| 293 |
+
|
| 294 |
+
assistant_message = {
|
| 295 |
+
"role": "assistant",
|
| 296 |
+
"content": assistant_content,
|
| 297 |
+
}
|
| 298 |
+
|
| 299 |
+
# Build user message with tool_result blocks
|
| 300 |
+
user_message = {
|
| 301 |
+
"role": "user",
|
| 302 |
+
"content": [
|
| 303 |
+
{
|
| 304 |
+
"type": "tool_result",
|
| 305 |
+
"tool_use_id": tool_calls[i]["id"],
|
| 306 |
+
"content": search_results[i],
|
| 307 |
+
}
|
| 308 |
+
for i in range(len(tool_calls))
|
| 309 |
+
],
|
| 310 |
+
}
|
| 311 |
+
|
| 312 |
+
return assistant_message, user_message
|
| 313 |
+
|
| 314 |
+
@staticmethod
|
| 315 |
+
def _transform_response_openai(
|
| 316 |
+
tool_calls: List[Dict],
|
| 317 |
+
search_results: List[str],
|
| 318 |
+
) -> Tuple[Dict, List[Dict]]:
|
| 319 |
+
"""Transform to OpenAI format (assistant with tool_calls, separate tool messages)"""
|
| 320 |
+
# Build assistant message with tool_calls
|
| 321 |
+
assistant_message = {
|
| 322 |
+
"role": "assistant",
|
| 323 |
+
"tool_calls": [
|
| 324 |
+
{
|
| 325 |
+
"id": tc["id"],
|
| 326 |
+
"type": "function",
|
| 327 |
+
"function": {
|
| 328 |
+
"name": tc["name"],
|
| 329 |
+
"arguments": json.dumps(tc["input"])
|
| 330 |
+
if isinstance(tc["input"], dict)
|
| 331 |
+
else str(tc["input"]),
|
| 332 |
+
},
|
| 333 |
+
}
|
| 334 |
+
for tc in tool_calls
|
| 335 |
+
],
|
| 336 |
+
}
|
| 337 |
+
|
| 338 |
+
# Build separate tool messages (one per tool call)
|
| 339 |
+
tool_messages = [
|
| 340 |
+
{
|
| 341 |
+
"role": "tool",
|
| 342 |
+
"tool_call_id": tool_calls[i]["id"],
|
| 343 |
+
"content": search_results[i],
|
| 344 |
+
}
|
| 345 |
+
for i in range(len(tool_calls))
|
| 346 |
+
]
|
| 347 |
+
|
| 348 |
+
return assistant_message, tool_messages
|
| 349 |
+
|
| 350 |
+
@staticmethod
|
| 351 |
+
def format_search_response(result: SearchResponse) -> str:
|
| 352 |
+
"""
|
| 353 |
+
Format SearchResponse as text for tool_result content.
|
| 354 |
+
|
| 355 |
+
Args:
|
| 356 |
+
result: SearchResponse from litellm.asearch()
|
| 357 |
+
|
| 358 |
+
Returns:
|
| 359 |
+
Formatted text with Title, URL, Snippet for each result
|
| 360 |
+
"""
|
| 361 |
+
# Convert SearchResponse to string
|
| 362 |
+
if hasattr(result, "results") and result.results:
|
| 363 |
+
# Format results as text
|
| 364 |
+
search_result_text = "\n\n".join(
|
| 365 |
+
[
|
| 366 |
+
f"Title: {r.title}\nURL: {r.url}\nSnippet: {r.snippet}"
|
| 367 |
+
for r in result.results
|
| 368 |
+
]
|
| 369 |
+
)
|
| 370 |
+
else:
|
| 371 |
+
search_result_text = str(result)
|
| 372 |
+
|
| 373 |
+
return search_result_text
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/weights_biases.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
imported_openAIResponse = True
|
| 2 |
+
try:
|
| 3 |
+
import io
|
| 4 |
+
import logging
|
| 5 |
+
import sys
|
| 6 |
+
from typing import Any, Dict, List, Optional, TypeVar
|
| 7 |
+
|
| 8 |
+
from wandb.sdk.data_types import trace_tree
|
| 9 |
+
|
| 10 |
+
if sys.version_info >= (3, 8):
|
| 11 |
+
from typing import Literal, Protocol
|
| 12 |
+
else:
|
| 13 |
+
from typing_extensions import Literal, Protocol
|
| 14 |
+
|
| 15 |
+
logger = logging.getLogger(__name__)
|
| 16 |
+
|
| 17 |
+
K = TypeVar("K", bound=str)
|
| 18 |
+
V = TypeVar("V")
|
| 19 |
+
|
| 20 |
+
class OpenAIResponse(Protocol[K, V]): # type: ignore
|
| 21 |
+
# contains a (known) object attribute
|
| 22 |
+
object: Literal["chat.completion", "edit", "text_completion"]
|
| 23 |
+
|
| 24 |
+
def __getitem__(self, key: K) -> V:
|
| 25 |
+
... # noqa
|
| 26 |
+
|
| 27 |
+
def get(self, key: K, default: Optional[V] = None) -> Optional[V]: # noqa
|
| 28 |
+
... # pragma: no cover
|
| 29 |
+
|
| 30 |
+
class OpenAIRequestResponseResolver:
|
| 31 |
+
def __call__(
|
| 32 |
+
self,
|
| 33 |
+
request: Dict[str, Any],
|
| 34 |
+
response: OpenAIResponse,
|
| 35 |
+
time_elapsed: float,
|
| 36 |
+
) -> Optional[trace_tree.WBTraceTree]:
|
| 37 |
+
try:
|
| 38 |
+
if response["object"] == "edit":
|
| 39 |
+
return self._resolve_edit(request, response, time_elapsed)
|
| 40 |
+
elif response["object"] == "text_completion":
|
| 41 |
+
return self._resolve_completion(request, response, time_elapsed)
|
| 42 |
+
elif response["object"] == "chat.completion":
|
| 43 |
+
return self._resolve_chat_completion(
|
| 44 |
+
request, response, time_elapsed
|
| 45 |
+
)
|
| 46 |
+
else:
|
| 47 |
+
logger.debug(
|
| 48 |
+
f"Unknown OpenAI response object: {response['object']}"
|
| 49 |
+
)
|
| 50 |
+
except Exception as e:
|
| 51 |
+
logger.warning(f"Failed to resolve request/response: {e}")
|
| 52 |
+
return None
|
| 53 |
+
|
| 54 |
+
@staticmethod
|
| 55 |
+
def results_to_trace_tree(
|
| 56 |
+
request: Dict[str, Any],
|
| 57 |
+
response: OpenAIResponse,
|
| 58 |
+
results: List[trace_tree.Result],
|
| 59 |
+
time_elapsed: float,
|
| 60 |
+
) -> trace_tree.WBTraceTree:
|
| 61 |
+
"""Converts the request, response, and results into a trace tree.
|
| 62 |
+
|
| 63 |
+
params:
|
| 64 |
+
request: The request dictionary
|
| 65 |
+
response: The response object
|
| 66 |
+
results: A list of results object
|
| 67 |
+
time_elapsed: The time elapsed in seconds
|
| 68 |
+
returns:
|
| 69 |
+
A wandb trace tree object.
|
| 70 |
+
"""
|
| 71 |
+
start_time_ms = int(round(response["created"] * 1000))
|
| 72 |
+
end_time_ms = start_time_ms + int(round(time_elapsed * 1000))
|
| 73 |
+
span = trace_tree.Span(
|
| 74 |
+
name=f"{response.get('model', 'openai')}_{response['object']}_{response.get('created')}",
|
| 75 |
+
attributes=dict(response), # type: ignore
|
| 76 |
+
start_time_ms=start_time_ms,
|
| 77 |
+
end_time_ms=end_time_ms,
|
| 78 |
+
span_kind=trace_tree.SpanKind.LLM,
|
| 79 |
+
results=results,
|
| 80 |
+
)
|
| 81 |
+
model_obj = {"request": request, "response": response, "_kind": "openai"}
|
| 82 |
+
return trace_tree.WBTraceTree(root_span=span, model_dict=model_obj)
|
| 83 |
+
|
| 84 |
+
def _resolve_edit(
|
| 85 |
+
self,
|
| 86 |
+
request: Dict[str, Any],
|
| 87 |
+
response: OpenAIResponse,
|
| 88 |
+
time_elapsed: float,
|
| 89 |
+
) -> trace_tree.WBTraceTree:
|
| 90 |
+
"""Resolves the request and response objects for `openai.Edit`."""
|
| 91 |
+
request_str = (
|
| 92 |
+
f"\n\n**Instruction**: {request['instruction']}\n\n"
|
| 93 |
+
f"**Input**: {request['input']}\n"
|
| 94 |
+
)
|
| 95 |
+
choices = [
|
| 96 |
+
f"\n\n**Edited**: {choice['text']}\n" for choice in response["choices"]
|
| 97 |
+
]
|
| 98 |
+
|
| 99 |
+
return self._request_response_result_to_trace(
|
| 100 |
+
request=request,
|
| 101 |
+
response=response,
|
| 102 |
+
request_str=request_str,
|
| 103 |
+
choices=choices,
|
| 104 |
+
time_elapsed=time_elapsed,
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
def _resolve_completion(
|
| 108 |
+
self,
|
| 109 |
+
request: Dict[str, Any],
|
| 110 |
+
response: OpenAIResponse,
|
| 111 |
+
time_elapsed: float,
|
| 112 |
+
) -> trace_tree.WBTraceTree:
|
| 113 |
+
"""Resolves the request and response objects for `openai.Completion`."""
|
| 114 |
+
request_str = f"\n\n**Prompt**: {request['prompt']}\n"
|
| 115 |
+
choices = [
|
| 116 |
+
f"\n\n**Completion**: {choice['text']}\n"
|
| 117 |
+
for choice in response["choices"]
|
| 118 |
+
]
|
| 119 |
+
|
| 120 |
+
return self._request_response_result_to_trace(
|
| 121 |
+
request=request,
|
| 122 |
+
response=response,
|
| 123 |
+
request_str=request_str,
|
| 124 |
+
choices=choices,
|
| 125 |
+
time_elapsed=time_elapsed,
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
def _resolve_chat_completion(
|
| 129 |
+
self,
|
| 130 |
+
request: Dict[str, Any],
|
| 131 |
+
response: OpenAIResponse,
|
| 132 |
+
time_elapsed: float,
|
| 133 |
+
) -> trace_tree.WBTraceTree:
|
| 134 |
+
"""Resolves the request and response objects for `openai.Completion`."""
|
| 135 |
+
prompt = io.StringIO()
|
| 136 |
+
for message in request["messages"]:
|
| 137 |
+
prompt.write(f"\n\n**{message['role']}**: {message['content']}\n")
|
| 138 |
+
request_str = prompt.getvalue()
|
| 139 |
+
|
| 140 |
+
choices = [
|
| 141 |
+
f"\n\n**{choice['message']['role']}**: {choice['message']['content']}\n"
|
| 142 |
+
for choice in response["choices"]
|
| 143 |
+
]
|
| 144 |
+
|
| 145 |
+
return self._request_response_result_to_trace(
|
| 146 |
+
request=request,
|
| 147 |
+
response=response,
|
| 148 |
+
request_str=request_str,
|
| 149 |
+
choices=choices,
|
| 150 |
+
time_elapsed=time_elapsed,
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
def _request_response_result_to_trace(
|
| 154 |
+
self,
|
| 155 |
+
request: Dict[str, Any],
|
| 156 |
+
response: OpenAIResponse,
|
| 157 |
+
request_str: str,
|
| 158 |
+
choices: List[str],
|
| 159 |
+
time_elapsed: float,
|
| 160 |
+
) -> trace_tree.WBTraceTree:
|
| 161 |
+
"""Resolves the request and response objects for `openai.Completion`."""
|
| 162 |
+
results = [
|
| 163 |
+
trace_tree.Result(
|
| 164 |
+
inputs={"request": request_str},
|
| 165 |
+
outputs={"response": choice},
|
| 166 |
+
)
|
| 167 |
+
for choice in choices
|
| 168 |
+
]
|
| 169 |
+
trace = self.results_to_trace_tree(request, response, results, time_elapsed)
|
| 170 |
+
return trace
|
| 171 |
+
|
| 172 |
+
except Exception:
|
| 173 |
+
imported_openAIResponse = False
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
#### What this does ####
|
| 177 |
+
# On success, logs events to Langfuse
|
| 178 |
+
import traceback
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
class WeightsBiasesLogger:
|
| 182 |
+
# Class variables or attributes
|
| 183 |
+
def __init__(self):
|
| 184 |
+
try:
|
| 185 |
+
pass
|
| 186 |
+
except Exception:
|
| 187 |
+
raise Exception(
|
| 188 |
+
"\033[91m wandb not installed, try running 'pip install wandb' to fix this error\033[0m"
|
| 189 |
+
)
|
| 190 |
+
if imported_openAIResponse is False:
|
| 191 |
+
raise Exception(
|
| 192 |
+
"\033[91m wandb not installed, try running 'pip install wandb' to fix this error\033[0m"
|
| 193 |
+
)
|
| 194 |
+
self.resolver = OpenAIRequestResponseResolver()
|
| 195 |
+
|
| 196 |
+
def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
|
| 197 |
+
# Method definition
|
| 198 |
+
import wandb
|
| 199 |
+
|
| 200 |
+
try:
|
| 201 |
+
print_verbose(f"W&B Logging - Enters logging function for model {kwargs}")
|
| 202 |
+
run = wandb.init()
|
| 203 |
+
print_verbose(response_obj)
|
| 204 |
+
|
| 205 |
+
trace = self.resolver(
|
| 206 |
+
kwargs, response_obj, (end_time - start_time).total_seconds()
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
if trace is not None and run is not None:
|
| 210 |
+
run.log({"trace": trace})
|
| 211 |
+
|
| 212 |
+
if run is not None:
|
| 213 |
+
run.finish()
|
| 214 |
+
print_verbose(
|
| 215 |
+
f"W&B Logging Logging - final response object: {response_obj}"
|
| 216 |
+
)
|
| 217 |
+
except Exception:
|
| 218 |
+
print_verbose(f"W&B Logging Layer Error - {traceback.format_exc()}")
|
| 219 |
+
pass
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/__init__.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
LiteLLM Interactions API
|
| 3 |
+
|
| 4 |
+
This module provides SDK methods for Google's Interactions API.
|
| 5 |
+
|
| 6 |
+
Usage:
|
| 7 |
+
import litellm
|
| 8 |
+
|
| 9 |
+
# Create an interaction with a model
|
| 10 |
+
response = litellm.interactions.create(
|
| 11 |
+
model="gemini-2.5-flash",
|
| 12 |
+
input="Hello, how are you?"
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
# Create an interaction with an agent
|
| 16 |
+
response = litellm.interactions.create(
|
| 17 |
+
agent="deep-research-pro-preview-12-2025",
|
| 18 |
+
input="Research the current state of cancer research"
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
# Async version
|
| 22 |
+
response = await litellm.interactions.acreate(...)
|
| 23 |
+
|
| 24 |
+
# Get an interaction
|
| 25 |
+
response = litellm.interactions.get(interaction_id="...")
|
| 26 |
+
|
| 27 |
+
# Delete an interaction
|
| 28 |
+
result = litellm.interactions.delete(interaction_id="...")
|
| 29 |
+
|
| 30 |
+
# Cancel an interaction
|
| 31 |
+
result = litellm.interactions.cancel(interaction_id="...")
|
| 32 |
+
|
| 33 |
+
Methods:
|
| 34 |
+
- create(): Sync create interaction
|
| 35 |
+
- acreate(): Async create interaction
|
| 36 |
+
- get(): Sync get interaction
|
| 37 |
+
- aget(): Async get interaction
|
| 38 |
+
- delete(): Sync delete interaction
|
| 39 |
+
- adelete(): Async delete interaction
|
| 40 |
+
- cancel(): Sync cancel interaction
|
| 41 |
+
- acancel(): Async cancel interaction
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
from litellm.interactions.main import (
|
| 45 |
+
acancel,
|
| 46 |
+
acreate,
|
| 47 |
+
adelete,
|
| 48 |
+
aget,
|
| 49 |
+
cancel,
|
| 50 |
+
create,
|
| 51 |
+
delete,
|
| 52 |
+
get,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
__all__ = [
|
| 56 |
+
# Create
|
| 57 |
+
"create",
|
| 58 |
+
"acreate",
|
| 59 |
+
# Get
|
| 60 |
+
"get",
|
| 61 |
+
"aget",
|
| 62 |
+
# Delete
|
| 63 |
+
"delete",
|
| 64 |
+
"adelete",
|
| 65 |
+
# Cancel
|
| 66 |
+
"cancel",
|
| 67 |
+
"acancel",
|
| 68 |
+
]
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/http_handler.py
ADDED
|
@@ -0,0 +1,697 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
HTTP Handler for Interactions API requests.
|
| 3 |
+
|
| 4 |
+
This module handles the HTTP communication for the Google Interactions API.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import (
|
| 8 |
+
Any,
|
| 9 |
+
AsyncIterator,
|
| 10 |
+
Coroutine,
|
| 11 |
+
Dict,
|
| 12 |
+
Iterator,
|
| 13 |
+
Optional,
|
| 14 |
+
Union,
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
import httpx
|
| 18 |
+
|
| 19 |
+
import litellm
|
| 20 |
+
from litellm.constants import request_timeout
|
| 21 |
+
from litellm.interactions.streaming_iterator import (
|
| 22 |
+
InteractionsAPIStreamingIterator,
|
| 23 |
+
SyncInteractionsAPIStreamingIterator,
|
| 24 |
+
)
|
| 25 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
| 26 |
+
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
|
| 27 |
+
from litellm.llms.custom_httpx.http_handler import (
|
| 28 |
+
AsyncHTTPHandler,
|
| 29 |
+
HTTPHandler,
|
| 30 |
+
_get_httpx_client,
|
| 31 |
+
get_async_httpx_client,
|
| 32 |
+
)
|
| 33 |
+
from litellm.types.interactions import (
|
| 34 |
+
CancelInteractionResult,
|
| 35 |
+
DeleteInteractionResult,
|
| 36 |
+
InteractionInput,
|
| 37 |
+
InteractionsAPIOptionalRequestParams,
|
| 38 |
+
InteractionsAPIResponse,
|
| 39 |
+
InteractionsAPIStreamingResponse,
|
| 40 |
+
)
|
| 41 |
+
from litellm.types.router import GenericLiteLLMParams
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class InteractionsHTTPHandler:
|
| 45 |
+
"""
|
| 46 |
+
HTTP handler for Interactions API requests.
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
def _handle_error(
|
| 50 |
+
self,
|
| 51 |
+
e: Exception,
|
| 52 |
+
provider_config: BaseInteractionsAPIConfig,
|
| 53 |
+
) -> Exception:
|
| 54 |
+
"""Handle errors from HTTP requests."""
|
| 55 |
+
if isinstance(e, httpx.HTTPStatusError):
|
| 56 |
+
error_message = e.response.text
|
| 57 |
+
status_code = e.response.status_code
|
| 58 |
+
headers = dict(e.response.headers)
|
| 59 |
+
return provider_config.get_error_class(
|
| 60 |
+
error_message=error_message,
|
| 61 |
+
status_code=status_code,
|
| 62 |
+
headers=headers,
|
| 63 |
+
)
|
| 64 |
+
return e
|
| 65 |
+
|
| 66 |
+
# =========================================================
|
| 67 |
+
# CREATE INTERACTION
|
| 68 |
+
# =========================================================
|
| 69 |
+
|
| 70 |
+
def create_interaction(
|
| 71 |
+
self,
|
| 72 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 73 |
+
optional_params: InteractionsAPIOptionalRequestParams,
|
| 74 |
+
custom_llm_provider: str,
|
| 75 |
+
litellm_params: GenericLiteLLMParams,
|
| 76 |
+
logging_obj: LiteLLMLoggingObj,
|
| 77 |
+
model: Optional[str] = None,
|
| 78 |
+
agent: Optional[str] = None,
|
| 79 |
+
input: Optional[InteractionInput] = None,
|
| 80 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 81 |
+
extra_body: Optional[Dict[str, Any]] = None,
|
| 82 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 83 |
+
client: Optional[HTTPHandler] = None,
|
| 84 |
+
_is_async: bool = False,
|
| 85 |
+
stream: Optional[bool] = None,
|
| 86 |
+
) -> Union[
|
| 87 |
+
InteractionsAPIResponse,
|
| 88 |
+
Iterator[InteractionsAPIStreamingResponse],
|
| 89 |
+
Coroutine[
|
| 90 |
+
Any,
|
| 91 |
+
Any,
|
| 92 |
+
Union[
|
| 93 |
+
InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]
|
| 94 |
+
],
|
| 95 |
+
],
|
| 96 |
+
]:
|
| 97 |
+
"""
|
| 98 |
+
Create a new interaction (synchronous or async based on _is_async flag).
|
| 99 |
+
|
| 100 |
+
Per Google's OpenAPI spec, the endpoint is POST /{api_version}/interactions
|
| 101 |
+
"""
|
| 102 |
+
if _is_async:
|
| 103 |
+
return self.async_create_interaction(
|
| 104 |
+
model=model,
|
| 105 |
+
agent=agent,
|
| 106 |
+
input=input,
|
| 107 |
+
interactions_api_config=interactions_api_config,
|
| 108 |
+
optional_params=optional_params,
|
| 109 |
+
custom_llm_provider=custom_llm_provider,
|
| 110 |
+
litellm_params=litellm_params,
|
| 111 |
+
logging_obj=logging_obj,
|
| 112 |
+
extra_headers=extra_headers,
|
| 113 |
+
extra_body=extra_body,
|
| 114 |
+
timeout=timeout,
|
| 115 |
+
stream=stream,
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
if client is None:
|
| 119 |
+
sync_httpx_client = _get_httpx_client(
|
| 120 |
+
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
|
| 121 |
+
)
|
| 122 |
+
else:
|
| 123 |
+
sync_httpx_client = client
|
| 124 |
+
|
| 125 |
+
headers = interactions_api_config.validate_environment(
|
| 126 |
+
headers=extra_headers or {},
|
| 127 |
+
model=model or "",
|
| 128 |
+
litellm_params=litellm_params,
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
api_base = interactions_api_config.get_complete_url(
|
| 132 |
+
api_base=litellm_params.api_base or "",
|
| 133 |
+
model=model,
|
| 134 |
+
agent=agent,
|
| 135 |
+
litellm_params=dict(litellm_params),
|
| 136 |
+
stream=stream,
|
| 137 |
+
)
|
| 138 |
+
|
| 139 |
+
data = interactions_api_config.transform_request(
|
| 140 |
+
model=model,
|
| 141 |
+
agent=agent,
|
| 142 |
+
input=input,
|
| 143 |
+
optional_params=optional_params,
|
| 144 |
+
litellm_params=litellm_params,
|
| 145 |
+
headers=headers,
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
if extra_body:
|
| 149 |
+
data.update(extra_body)
|
| 150 |
+
|
| 151 |
+
# Logging
|
| 152 |
+
logging_obj.pre_call(
|
| 153 |
+
input=input,
|
| 154 |
+
api_key="",
|
| 155 |
+
additional_args={
|
| 156 |
+
"complete_input_dict": data,
|
| 157 |
+
"api_base": api_base,
|
| 158 |
+
"headers": headers,
|
| 159 |
+
},
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
try:
|
| 163 |
+
if stream:
|
| 164 |
+
response = sync_httpx_client.post(
|
| 165 |
+
url=api_base,
|
| 166 |
+
headers=headers,
|
| 167 |
+
json=data,
|
| 168 |
+
timeout=timeout or request_timeout,
|
| 169 |
+
stream=True,
|
| 170 |
+
)
|
| 171 |
+
return self._create_sync_streaming_iterator(
|
| 172 |
+
response=response,
|
| 173 |
+
model=model,
|
| 174 |
+
logging_obj=logging_obj,
|
| 175 |
+
interactions_api_config=interactions_api_config,
|
| 176 |
+
)
|
| 177 |
+
else:
|
| 178 |
+
response = sync_httpx_client.post(
|
| 179 |
+
url=api_base,
|
| 180 |
+
headers=headers,
|
| 181 |
+
json=data,
|
| 182 |
+
timeout=timeout or request_timeout,
|
| 183 |
+
)
|
| 184 |
+
except Exception as e:
|
| 185 |
+
raise self._handle_error(e=e, provider_config=interactions_api_config)
|
| 186 |
+
|
| 187 |
+
return interactions_api_config.transform_response(
|
| 188 |
+
model=model,
|
| 189 |
+
raw_response=response,
|
| 190 |
+
logging_obj=logging_obj,
|
| 191 |
+
)
|
| 192 |
+
|
| 193 |
+
async def async_create_interaction(
|
| 194 |
+
self,
|
| 195 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 196 |
+
optional_params: InteractionsAPIOptionalRequestParams,
|
| 197 |
+
custom_llm_provider: str,
|
| 198 |
+
litellm_params: GenericLiteLLMParams,
|
| 199 |
+
logging_obj: LiteLLMLoggingObj,
|
| 200 |
+
model: Optional[str] = None,
|
| 201 |
+
agent: Optional[str] = None,
|
| 202 |
+
input: Optional[InteractionInput] = None,
|
| 203 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 204 |
+
extra_body: Optional[Dict[str, Any]] = None,
|
| 205 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 206 |
+
client: Optional[AsyncHTTPHandler] = None,
|
| 207 |
+
stream: Optional[bool] = None,
|
| 208 |
+
) -> Union[
|
| 209 |
+
InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]
|
| 210 |
+
]:
|
| 211 |
+
"""
|
| 212 |
+
Create a new interaction (async version).
|
| 213 |
+
"""
|
| 214 |
+
if client is None:
|
| 215 |
+
async_httpx_client = get_async_httpx_client(
|
| 216 |
+
llm_provider=litellm.LlmProviders(custom_llm_provider),
|
| 217 |
+
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
|
| 218 |
+
)
|
| 219 |
+
else:
|
| 220 |
+
async_httpx_client = client
|
| 221 |
+
|
| 222 |
+
headers = interactions_api_config.validate_environment(
|
| 223 |
+
headers=extra_headers or {},
|
| 224 |
+
model=model or "",
|
| 225 |
+
litellm_params=litellm_params,
|
| 226 |
+
)
|
| 227 |
+
|
| 228 |
+
api_base = interactions_api_config.get_complete_url(
|
| 229 |
+
api_base=litellm_params.api_base or "",
|
| 230 |
+
model=model,
|
| 231 |
+
agent=agent,
|
| 232 |
+
litellm_params=dict(litellm_params),
|
| 233 |
+
stream=stream,
|
| 234 |
+
)
|
| 235 |
+
|
| 236 |
+
data = interactions_api_config.transform_request(
|
| 237 |
+
model=model,
|
| 238 |
+
agent=agent,
|
| 239 |
+
input=input,
|
| 240 |
+
optional_params=optional_params,
|
| 241 |
+
litellm_params=litellm_params,
|
| 242 |
+
headers=headers,
|
| 243 |
+
)
|
| 244 |
+
|
| 245 |
+
if extra_body:
|
| 246 |
+
data.update(extra_body)
|
| 247 |
+
|
| 248 |
+
# Logging
|
| 249 |
+
logging_obj.pre_call(
|
| 250 |
+
input=input,
|
| 251 |
+
api_key="",
|
| 252 |
+
additional_args={
|
| 253 |
+
"complete_input_dict": data,
|
| 254 |
+
"api_base": api_base,
|
| 255 |
+
"headers": headers,
|
| 256 |
+
},
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
try:
|
| 260 |
+
if stream:
|
| 261 |
+
response = await async_httpx_client.post(
|
| 262 |
+
url=api_base,
|
| 263 |
+
headers=headers,
|
| 264 |
+
json=data,
|
| 265 |
+
timeout=timeout or request_timeout,
|
| 266 |
+
stream=True,
|
| 267 |
+
)
|
| 268 |
+
return self._create_async_streaming_iterator(
|
| 269 |
+
response=response,
|
| 270 |
+
model=model,
|
| 271 |
+
logging_obj=logging_obj,
|
| 272 |
+
interactions_api_config=interactions_api_config,
|
| 273 |
+
)
|
| 274 |
+
else:
|
| 275 |
+
response = await async_httpx_client.post(
|
| 276 |
+
url=api_base,
|
| 277 |
+
headers=headers,
|
| 278 |
+
json=data,
|
| 279 |
+
timeout=timeout or request_timeout,
|
| 280 |
+
)
|
| 281 |
+
except Exception as e:
|
| 282 |
+
raise self._handle_error(e=e, provider_config=interactions_api_config)
|
| 283 |
+
|
| 284 |
+
return interactions_api_config.transform_response(
|
| 285 |
+
model=model,
|
| 286 |
+
raw_response=response,
|
| 287 |
+
logging_obj=logging_obj,
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
def _create_sync_streaming_iterator(
|
| 291 |
+
self,
|
| 292 |
+
response: httpx.Response,
|
| 293 |
+
model: Optional[str],
|
| 294 |
+
logging_obj: LiteLLMLoggingObj,
|
| 295 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 296 |
+
) -> SyncInteractionsAPIStreamingIterator:
|
| 297 |
+
"""Create a synchronous streaming iterator.
|
| 298 |
+
|
| 299 |
+
Google AI's streaming format uses SSE (Server-Sent Events).
|
| 300 |
+
Returns a proper streaming iterator that yields chunks as they arrive.
|
| 301 |
+
"""
|
| 302 |
+
return SyncInteractionsAPIStreamingIterator(
|
| 303 |
+
response=response,
|
| 304 |
+
model=model,
|
| 305 |
+
interactions_api_config=interactions_api_config,
|
| 306 |
+
logging_obj=logging_obj,
|
| 307 |
+
)
|
| 308 |
+
|
| 309 |
+
def _create_async_streaming_iterator(
|
| 310 |
+
self,
|
| 311 |
+
response: httpx.Response,
|
| 312 |
+
model: Optional[str],
|
| 313 |
+
logging_obj: LiteLLMLoggingObj,
|
| 314 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 315 |
+
) -> InteractionsAPIStreamingIterator:
|
| 316 |
+
"""Create an asynchronous streaming iterator.
|
| 317 |
+
|
| 318 |
+
Google AI's streaming format uses SSE (Server-Sent Events).
|
| 319 |
+
Returns a proper streaming iterator that yields chunks as they arrive.
|
| 320 |
+
"""
|
| 321 |
+
return InteractionsAPIStreamingIterator(
|
| 322 |
+
response=response,
|
| 323 |
+
model=model,
|
| 324 |
+
interactions_api_config=interactions_api_config,
|
| 325 |
+
logging_obj=logging_obj,
|
| 326 |
+
)
|
| 327 |
+
|
| 328 |
+
# =========================================================
|
| 329 |
+
# GET INTERACTION
|
| 330 |
+
# =========================================================
|
| 331 |
+
|
| 332 |
+
def get_interaction(
|
| 333 |
+
self,
|
| 334 |
+
interaction_id: str,
|
| 335 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 336 |
+
custom_llm_provider: str,
|
| 337 |
+
litellm_params: GenericLiteLLMParams,
|
| 338 |
+
logging_obj: LiteLLMLoggingObj,
|
| 339 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 340 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 341 |
+
client: Optional[HTTPHandler] = None,
|
| 342 |
+
_is_async: bool = False,
|
| 343 |
+
) -> Union[InteractionsAPIResponse, Coroutine[Any, Any, InteractionsAPIResponse]]:
|
| 344 |
+
"""Get an interaction by ID."""
|
| 345 |
+
if _is_async:
|
| 346 |
+
return self.async_get_interaction(
|
| 347 |
+
interaction_id=interaction_id,
|
| 348 |
+
interactions_api_config=interactions_api_config,
|
| 349 |
+
custom_llm_provider=custom_llm_provider,
|
| 350 |
+
litellm_params=litellm_params,
|
| 351 |
+
logging_obj=logging_obj,
|
| 352 |
+
extra_headers=extra_headers,
|
| 353 |
+
timeout=timeout,
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
if client is None:
|
| 357 |
+
sync_httpx_client = _get_httpx_client(
|
| 358 |
+
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
|
| 359 |
+
)
|
| 360 |
+
else:
|
| 361 |
+
sync_httpx_client = client
|
| 362 |
+
|
| 363 |
+
headers = interactions_api_config.validate_environment(
|
| 364 |
+
headers=extra_headers or {},
|
| 365 |
+
model="",
|
| 366 |
+
litellm_params=litellm_params,
|
| 367 |
+
)
|
| 368 |
+
|
| 369 |
+
url, params = interactions_api_config.transform_get_interaction_request(
|
| 370 |
+
interaction_id=interaction_id,
|
| 371 |
+
api_base=litellm_params.api_base or "",
|
| 372 |
+
litellm_params=litellm_params,
|
| 373 |
+
headers=headers,
|
| 374 |
+
)
|
| 375 |
+
|
| 376 |
+
logging_obj.pre_call(
|
| 377 |
+
input=interaction_id,
|
| 378 |
+
api_key="",
|
| 379 |
+
additional_args={"api_base": url, "headers": headers},
|
| 380 |
+
)
|
| 381 |
+
|
| 382 |
+
try:
|
| 383 |
+
response = sync_httpx_client.get(
|
| 384 |
+
url=url,
|
| 385 |
+
headers=headers,
|
| 386 |
+
params=params,
|
| 387 |
+
)
|
| 388 |
+
except Exception as e:
|
| 389 |
+
raise self._handle_error(e=e, provider_config=interactions_api_config)
|
| 390 |
+
|
| 391 |
+
return interactions_api_config.transform_get_interaction_response(
|
| 392 |
+
raw_response=response,
|
| 393 |
+
logging_obj=logging_obj,
|
| 394 |
+
)
|
| 395 |
+
|
| 396 |
+
async def async_get_interaction(
|
| 397 |
+
self,
|
| 398 |
+
interaction_id: str,
|
| 399 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 400 |
+
custom_llm_provider: str,
|
| 401 |
+
litellm_params: GenericLiteLLMParams,
|
| 402 |
+
logging_obj: LiteLLMLoggingObj,
|
| 403 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 404 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 405 |
+
client: Optional[AsyncHTTPHandler] = None,
|
| 406 |
+
) -> InteractionsAPIResponse:
|
| 407 |
+
"""Get an interaction by ID (async version)."""
|
| 408 |
+
if client is None:
|
| 409 |
+
async_httpx_client = get_async_httpx_client(
|
| 410 |
+
llm_provider=litellm.LlmProviders(custom_llm_provider),
|
| 411 |
+
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
|
| 412 |
+
)
|
| 413 |
+
else:
|
| 414 |
+
async_httpx_client = client
|
| 415 |
+
|
| 416 |
+
headers = interactions_api_config.validate_environment(
|
| 417 |
+
headers=extra_headers or {},
|
| 418 |
+
model="",
|
| 419 |
+
litellm_params=litellm_params,
|
| 420 |
+
)
|
| 421 |
+
|
| 422 |
+
url, params = interactions_api_config.transform_get_interaction_request(
|
| 423 |
+
interaction_id=interaction_id,
|
| 424 |
+
api_base=litellm_params.api_base or "",
|
| 425 |
+
litellm_params=litellm_params,
|
| 426 |
+
headers=headers,
|
| 427 |
+
)
|
| 428 |
+
|
| 429 |
+
logging_obj.pre_call(
|
| 430 |
+
input=interaction_id,
|
| 431 |
+
api_key="",
|
| 432 |
+
additional_args={"api_base": url, "headers": headers},
|
| 433 |
+
)
|
| 434 |
+
|
| 435 |
+
try:
|
| 436 |
+
response = await async_httpx_client.get(
|
| 437 |
+
url=url,
|
| 438 |
+
headers=headers,
|
| 439 |
+
params=params,
|
| 440 |
+
)
|
| 441 |
+
except Exception as e:
|
| 442 |
+
raise self._handle_error(e=e, provider_config=interactions_api_config)
|
| 443 |
+
|
| 444 |
+
return interactions_api_config.transform_get_interaction_response(
|
| 445 |
+
raw_response=response,
|
| 446 |
+
logging_obj=logging_obj,
|
| 447 |
+
)
|
| 448 |
+
|
| 449 |
+
# =========================================================
|
| 450 |
+
# DELETE INTERACTION
|
| 451 |
+
# =========================================================
|
| 452 |
+
|
| 453 |
+
def delete_interaction(
|
| 454 |
+
self,
|
| 455 |
+
interaction_id: str,
|
| 456 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 457 |
+
custom_llm_provider: str,
|
| 458 |
+
litellm_params: GenericLiteLLMParams,
|
| 459 |
+
logging_obj: LiteLLMLoggingObj,
|
| 460 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 461 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 462 |
+
client: Optional[HTTPHandler] = None,
|
| 463 |
+
_is_async: bool = False,
|
| 464 |
+
) -> Union[DeleteInteractionResult, Coroutine[Any, Any, DeleteInteractionResult]]:
|
| 465 |
+
"""Delete an interaction by ID."""
|
| 466 |
+
if _is_async:
|
| 467 |
+
return self.async_delete_interaction(
|
| 468 |
+
interaction_id=interaction_id,
|
| 469 |
+
interactions_api_config=interactions_api_config,
|
| 470 |
+
custom_llm_provider=custom_llm_provider,
|
| 471 |
+
litellm_params=litellm_params,
|
| 472 |
+
logging_obj=logging_obj,
|
| 473 |
+
extra_headers=extra_headers,
|
| 474 |
+
timeout=timeout,
|
| 475 |
+
)
|
| 476 |
+
|
| 477 |
+
if client is None:
|
| 478 |
+
sync_httpx_client = _get_httpx_client(
|
| 479 |
+
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
|
| 480 |
+
)
|
| 481 |
+
else:
|
| 482 |
+
sync_httpx_client = client
|
| 483 |
+
|
| 484 |
+
headers = interactions_api_config.validate_environment(
|
| 485 |
+
headers=extra_headers or {},
|
| 486 |
+
model="",
|
| 487 |
+
litellm_params=litellm_params,
|
| 488 |
+
)
|
| 489 |
+
|
| 490 |
+
url, data = interactions_api_config.transform_delete_interaction_request(
|
| 491 |
+
interaction_id=interaction_id,
|
| 492 |
+
api_base=litellm_params.api_base or "",
|
| 493 |
+
litellm_params=litellm_params,
|
| 494 |
+
headers=headers,
|
| 495 |
+
)
|
| 496 |
+
|
| 497 |
+
logging_obj.pre_call(
|
| 498 |
+
input=interaction_id,
|
| 499 |
+
api_key="",
|
| 500 |
+
additional_args={"api_base": url, "headers": headers},
|
| 501 |
+
)
|
| 502 |
+
|
| 503 |
+
try:
|
| 504 |
+
response = sync_httpx_client.delete(
|
| 505 |
+
url=url,
|
| 506 |
+
headers=headers,
|
| 507 |
+
timeout=timeout or request_timeout,
|
| 508 |
+
)
|
| 509 |
+
except Exception as e:
|
| 510 |
+
raise self._handle_error(e=e, provider_config=interactions_api_config)
|
| 511 |
+
|
| 512 |
+
return interactions_api_config.transform_delete_interaction_response(
|
| 513 |
+
raw_response=response,
|
| 514 |
+
logging_obj=logging_obj,
|
| 515 |
+
interaction_id=interaction_id,
|
| 516 |
+
)
|
| 517 |
+
|
| 518 |
+
async def async_delete_interaction(
|
| 519 |
+
self,
|
| 520 |
+
interaction_id: str,
|
| 521 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 522 |
+
custom_llm_provider: str,
|
| 523 |
+
litellm_params: GenericLiteLLMParams,
|
| 524 |
+
logging_obj: LiteLLMLoggingObj,
|
| 525 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 526 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 527 |
+
client: Optional[AsyncHTTPHandler] = None,
|
| 528 |
+
) -> DeleteInteractionResult:
|
| 529 |
+
"""Delete an interaction by ID (async version)."""
|
| 530 |
+
if client is None:
|
| 531 |
+
async_httpx_client = get_async_httpx_client(
|
| 532 |
+
llm_provider=litellm.LlmProviders(custom_llm_provider),
|
| 533 |
+
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
|
| 534 |
+
)
|
| 535 |
+
else:
|
| 536 |
+
async_httpx_client = client
|
| 537 |
+
|
| 538 |
+
headers = interactions_api_config.validate_environment(
|
| 539 |
+
headers=extra_headers or {},
|
| 540 |
+
model="",
|
| 541 |
+
litellm_params=litellm_params,
|
| 542 |
+
)
|
| 543 |
+
|
| 544 |
+
url, data = interactions_api_config.transform_delete_interaction_request(
|
| 545 |
+
interaction_id=interaction_id,
|
| 546 |
+
api_base=litellm_params.api_base or "",
|
| 547 |
+
litellm_params=litellm_params,
|
| 548 |
+
headers=headers,
|
| 549 |
+
)
|
| 550 |
+
|
| 551 |
+
logging_obj.pre_call(
|
| 552 |
+
input=interaction_id,
|
| 553 |
+
api_key="",
|
| 554 |
+
additional_args={"api_base": url, "headers": headers},
|
| 555 |
+
)
|
| 556 |
+
|
| 557 |
+
try:
|
| 558 |
+
response = await async_httpx_client.delete(
|
| 559 |
+
url=url,
|
| 560 |
+
headers=headers,
|
| 561 |
+
timeout=timeout or request_timeout,
|
| 562 |
+
)
|
| 563 |
+
except Exception as e:
|
| 564 |
+
raise self._handle_error(e=e, provider_config=interactions_api_config)
|
| 565 |
+
|
| 566 |
+
return interactions_api_config.transform_delete_interaction_response(
|
| 567 |
+
raw_response=response,
|
| 568 |
+
logging_obj=logging_obj,
|
| 569 |
+
interaction_id=interaction_id,
|
| 570 |
+
)
|
| 571 |
+
|
| 572 |
+
# =========================================================
|
| 573 |
+
# CANCEL INTERACTION
|
| 574 |
+
# =========================================================
|
| 575 |
+
|
| 576 |
+
def cancel_interaction(
|
| 577 |
+
self,
|
| 578 |
+
interaction_id: str,
|
| 579 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 580 |
+
custom_llm_provider: str,
|
| 581 |
+
litellm_params: GenericLiteLLMParams,
|
| 582 |
+
logging_obj: LiteLLMLoggingObj,
|
| 583 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 584 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 585 |
+
client: Optional[HTTPHandler] = None,
|
| 586 |
+
_is_async: bool = False,
|
| 587 |
+
) -> Union[CancelInteractionResult, Coroutine[Any, Any, CancelInteractionResult]]:
|
| 588 |
+
"""Cancel an interaction by ID."""
|
| 589 |
+
if _is_async:
|
| 590 |
+
return self.async_cancel_interaction(
|
| 591 |
+
interaction_id=interaction_id,
|
| 592 |
+
interactions_api_config=interactions_api_config,
|
| 593 |
+
custom_llm_provider=custom_llm_provider,
|
| 594 |
+
litellm_params=litellm_params,
|
| 595 |
+
logging_obj=logging_obj,
|
| 596 |
+
extra_headers=extra_headers,
|
| 597 |
+
timeout=timeout,
|
| 598 |
+
)
|
| 599 |
+
|
| 600 |
+
if client is None:
|
| 601 |
+
sync_httpx_client = _get_httpx_client(
|
| 602 |
+
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
|
| 603 |
+
)
|
| 604 |
+
else:
|
| 605 |
+
sync_httpx_client = client
|
| 606 |
+
|
| 607 |
+
headers = interactions_api_config.validate_environment(
|
| 608 |
+
headers=extra_headers or {},
|
| 609 |
+
model="",
|
| 610 |
+
litellm_params=litellm_params,
|
| 611 |
+
)
|
| 612 |
+
|
| 613 |
+
url, data = interactions_api_config.transform_cancel_interaction_request(
|
| 614 |
+
interaction_id=interaction_id,
|
| 615 |
+
api_base=litellm_params.api_base or "",
|
| 616 |
+
litellm_params=litellm_params,
|
| 617 |
+
headers=headers,
|
| 618 |
+
)
|
| 619 |
+
|
| 620 |
+
logging_obj.pre_call(
|
| 621 |
+
input=interaction_id,
|
| 622 |
+
api_key="",
|
| 623 |
+
additional_args={"api_base": url, "headers": headers},
|
| 624 |
+
)
|
| 625 |
+
|
| 626 |
+
try:
|
| 627 |
+
response = sync_httpx_client.post(
|
| 628 |
+
url=url,
|
| 629 |
+
headers=headers,
|
| 630 |
+
json=data,
|
| 631 |
+
timeout=timeout or request_timeout,
|
| 632 |
+
)
|
| 633 |
+
except Exception as e:
|
| 634 |
+
raise self._handle_error(e=e, provider_config=interactions_api_config)
|
| 635 |
+
|
| 636 |
+
return interactions_api_config.transform_cancel_interaction_response(
|
| 637 |
+
raw_response=response,
|
| 638 |
+
logging_obj=logging_obj,
|
| 639 |
+
)
|
| 640 |
+
|
| 641 |
+
async def async_cancel_interaction(
|
| 642 |
+
self,
|
| 643 |
+
interaction_id: str,
|
| 644 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 645 |
+
custom_llm_provider: str,
|
| 646 |
+
litellm_params: GenericLiteLLMParams,
|
| 647 |
+
logging_obj: LiteLLMLoggingObj,
|
| 648 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 649 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 650 |
+
client: Optional[AsyncHTTPHandler] = None,
|
| 651 |
+
) -> CancelInteractionResult:
|
| 652 |
+
"""Cancel an interaction by ID (async version)."""
|
| 653 |
+
if client is None:
|
| 654 |
+
async_httpx_client = get_async_httpx_client(
|
| 655 |
+
llm_provider=litellm.LlmProviders(custom_llm_provider),
|
| 656 |
+
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
|
| 657 |
+
)
|
| 658 |
+
else:
|
| 659 |
+
async_httpx_client = client
|
| 660 |
+
|
| 661 |
+
headers = interactions_api_config.validate_environment(
|
| 662 |
+
headers=extra_headers or {},
|
| 663 |
+
model="",
|
| 664 |
+
litellm_params=litellm_params,
|
| 665 |
+
)
|
| 666 |
+
|
| 667 |
+
url, data = interactions_api_config.transform_cancel_interaction_request(
|
| 668 |
+
interaction_id=interaction_id,
|
| 669 |
+
api_base=litellm_params.api_base or "",
|
| 670 |
+
litellm_params=litellm_params,
|
| 671 |
+
headers=headers,
|
| 672 |
+
)
|
| 673 |
+
|
| 674 |
+
logging_obj.pre_call(
|
| 675 |
+
input=interaction_id,
|
| 676 |
+
api_key="",
|
| 677 |
+
additional_args={"api_base": url, "headers": headers},
|
| 678 |
+
)
|
| 679 |
+
|
| 680 |
+
try:
|
| 681 |
+
response = await async_httpx_client.post(
|
| 682 |
+
url=url,
|
| 683 |
+
headers=headers,
|
| 684 |
+
json=data,
|
| 685 |
+
timeout=timeout or request_timeout,
|
| 686 |
+
)
|
| 687 |
+
except Exception as e:
|
| 688 |
+
raise self._handle_error(e=e, provider_config=interactions_api_config)
|
| 689 |
+
|
| 690 |
+
return interactions_api_config.transform_cancel_interaction_response(
|
| 691 |
+
raw_response=response,
|
| 692 |
+
logging_obj=logging_obj,
|
| 693 |
+
)
|
| 694 |
+
|
| 695 |
+
|
| 696 |
+
# Initialize the HTTP handler singleton
|
| 697 |
+
interactions_http_handler = InteractionsHTTPHandler()
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Bridge module for connecting Interactions API to Responses API via litellm.responses().
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from litellm.interactions.litellm_responses_transformation.handler import (
|
| 6 |
+
LiteLLMResponsesInteractionsHandler,
|
| 7 |
+
)
|
| 8 |
+
from litellm.interactions.litellm_responses_transformation.transformation import (
|
| 9 |
+
LiteLLMResponsesInteractionsConfig,
|
| 10 |
+
)
|
| 11 |
+
|
| 12 |
+
__all__ = [
|
| 13 |
+
"LiteLLMResponsesInteractionsHandler",
|
| 14 |
+
"LiteLLMResponsesInteractionsConfig", # Transformation config class (not BaseInteractionsAPIConfig)
|
| 15 |
+
]
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/handler.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Handler for transforming interactions API requests to litellm.responses requests.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from typing import (
|
| 6 |
+
Any,
|
| 7 |
+
AsyncIterator,
|
| 8 |
+
Coroutine,
|
| 9 |
+
Dict,
|
| 10 |
+
Iterator,
|
| 11 |
+
Optional,
|
| 12 |
+
Union,
|
| 13 |
+
cast,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
import litellm
|
| 17 |
+
from litellm.interactions.litellm_responses_transformation.streaming_iterator import (
|
| 18 |
+
LiteLLMResponsesInteractionsStreamingIterator,
|
| 19 |
+
)
|
| 20 |
+
from litellm.interactions.litellm_responses_transformation.transformation import (
|
| 21 |
+
LiteLLMResponsesInteractionsConfig,
|
| 22 |
+
)
|
| 23 |
+
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
|
| 24 |
+
from litellm.types.interactions import (
|
| 25 |
+
InteractionInput,
|
| 26 |
+
InteractionsAPIOptionalRequestParams,
|
| 27 |
+
InteractionsAPIResponse,
|
| 28 |
+
InteractionsAPIStreamingResponse,
|
| 29 |
+
)
|
| 30 |
+
from litellm.types.llms.openai import ResponsesAPIResponse
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class LiteLLMResponsesInteractionsHandler:
|
| 34 |
+
"""Handler for bridging Interactions API to Responses API via litellm.responses()."""
|
| 35 |
+
|
| 36 |
+
def interactions_api_handler(
|
| 37 |
+
self,
|
| 38 |
+
model: str,
|
| 39 |
+
input: Optional[InteractionInput],
|
| 40 |
+
optional_params: InteractionsAPIOptionalRequestParams,
|
| 41 |
+
custom_llm_provider: Optional[str] = None,
|
| 42 |
+
_is_async: bool = False,
|
| 43 |
+
stream: Optional[bool] = None,
|
| 44 |
+
**kwargs,
|
| 45 |
+
) -> Union[
|
| 46 |
+
InteractionsAPIResponse,
|
| 47 |
+
Iterator[InteractionsAPIStreamingResponse],
|
| 48 |
+
Coroutine[
|
| 49 |
+
Any,
|
| 50 |
+
Any,
|
| 51 |
+
Union[
|
| 52 |
+
InteractionsAPIResponse,
|
| 53 |
+
AsyncIterator[InteractionsAPIStreamingResponse],
|
| 54 |
+
],
|
| 55 |
+
],
|
| 56 |
+
]:
|
| 57 |
+
"""
|
| 58 |
+
Handle Interactions API request by calling litellm.responses().
|
| 59 |
+
|
| 60 |
+
Args:
|
| 61 |
+
model: The model to use
|
| 62 |
+
input: The input content
|
| 63 |
+
optional_params: Optional parameters for the request
|
| 64 |
+
custom_llm_provider: Override LLM provider
|
| 65 |
+
_is_async: Whether this is an async call
|
| 66 |
+
stream: Whether to stream the response
|
| 67 |
+
**kwargs: Additional parameters
|
| 68 |
+
|
| 69 |
+
Returns:
|
| 70 |
+
InteractionsAPIResponse or streaming iterator
|
| 71 |
+
"""
|
| 72 |
+
# Transform interactions request to responses request
|
| 73 |
+
responses_request = LiteLLMResponsesInteractionsConfig.transform_interactions_request_to_responses_request(
|
| 74 |
+
model=model,
|
| 75 |
+
input=input,
|
| 76 |
+
optional_params=optional_params,
|
| 77 |
+
custom_llm_provider=custom_llm_provider,
|
| 78 |
+
stream=stream,
|
| 79 |
+
**kwargs,
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
if _is_async:
|
| 83 |
+
return self.async_interactions_api_handler(
|
| 84 |
+
responses_request=responses_request,
|
| 85 |
+
model=model,
|
| 86 |
+
input=input,
|
| 87 |
+
optional_params=optional_params,
|
| 88 |
+
**kwargs,
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
# Call litellm.responses()
|
| 92 |
+
# Note: litellm.responses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]
|
| 93 |
+
# but the type checker may see it as a coroutine in some contexts
|
| 94 |
+
responses_response = litellm.responses(
|
| 95 |
+
**responses_request,
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
# Handle streaming response
|
| 99 |
+
if isinstance(responses_response, BaseResponsesAPIStreamingIterator):
|
| 100 |
+
return LiteLLMResponsesInteractionsStreamingIterator(
|
| 101 |
+
model=model,
|
| 102 |
+
litellm_custom_stream_wrapper=responses_response,
|
| 103 |
+
request_input=input,
|
| 104 |
+
optional_params=optional_params,
|
| 105 |
+
custom_llm_provider=custom_llm_provider,
|
| 106 |
+
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
# At this point, responses_response must be ResponsesAPIResponse (not streaming)
|
| 110 |
+
# Cast to satisfy type checker since we've already checked it's not a streaming iterator
|
| 111 |
+
responses_api_response = cast(ResponsesAPIResponse, responses_response)
|
| 112 |
+
|
| 113 |
+
# Transform responses response to interactions response
|
| 114 |
+
return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
|
| 115 |
+
responses_response=responses_api_response,
|
| 116 |
+
model=model,
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
async def async_interactions_api_handler(
|
| 120 |
+
self,
|
| 121 |
+
responses_request: Dict[str, Any],
|
| 122 |
+
model: str,
|
| 123 |
+
input: Optional[InteractionInput],
|
| 124 |
+
optional_params: InteractionsAPIOptionalRequestParams,
|
| 125 |
+
**kwargs,
|
| 126 |
+
) -> Union[
|
| 127 |
+
InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]
|
| 128 |
+
]:
|
| 129 |
+
"""Async handler for interactions API requests."""
|
| 130 |
+
# Call litellm.aresponses()
|
| 131 |
+
# Note: litellm.aresponses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]
|
| 132 |
+
responses_response = await litellm.aresponses(
|
| 133 |
+
**responses_request,
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
# Handle streaming response
|
| 137 |
+
if isinstance(responses_response, BaseResponsesAPIStreamingIterator):
|
| 138 |
+
return LiteLLMResponsesInteractionsStreamingIterator(
|
| 139 |
+
model=model,
|
| 140 |
+
litellm_custom_stream_wrapper=responses_response,
|
| 141 |
+
request_input=input,
|
| 142 |
+
optional_params=optional_params,
|
| 143 |
+
custom_llm_provider=responses_request.get("custom_llm_provider"),
|
| 144 |
+
litellm_metadata=kwargs.get("litellm_metadata", {}),
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
+
# At this point, responses_response must be ResponsesAPIResponse (not streaming)
|
| 148 |
+
# Cast to satisfy type checker since we've already checked it's not a streaming iterator
|
| 149 |
+
responses_api_response = cast(ResponsesAPIResponse, responses_response)
|
| 150 |
+
|
| 151 |
+
# Transform responses response to interactions response
|
| 152 |
+
return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
|
| 153 |
+
responses_response=responses_api_response,
|
| 154 |
+
model=model,
|
| 155 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/streaming_iterator.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Streaming iterator for transforming Responses API stream to Interactions API stream.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from typing import Any, AsyncIterator, Dict, Iterator, Optional, cast
|
| 6 |
+
|
| 7 |
+
from litellm.responses.streaming_iterator import (
|
| 8 |
+
BaseResponsesAPIStreamingIterator,
|
| 9 |
+
ResponsesAPIStreamingIterator,
|
| 10 |
+
SyncResponsesAPIStreamingIterator,
|
| 11 |
+
)
|
| 12 |
+
from litellm.types.interactions import (
|
| 13 |
+
InteractionInput,
|
| 14 |
+
InteractionsAPIOptionalRequestParams,
|
| 15 |
+
InteractionsAPIStreamingResponse,
|
| 16 |
+
)
|
| 17 |
+
from litellm.types.llms.openai import (
|
| 18 |
+
OutputTextDeltaEvent,
|
| 19 |
+
ResponseCompletedEvent,
|
| 20 |
+
ResponseCreatedEvent,
|
| 21 |
+
ResponseInProgressEvent,
|
| 22 |
+
ResponsesAPIStreamingResponse,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
class LiteLLMResponsesInteractionsStreamingIterator:
|
| 27 |
+
"""
|
| 28 |
+
Iterator that wraps Responses API streaming and transforms chunks to Interactions API format.
|
| 29 |
+
|
| 30 |
+
This class handles both sync and async iteration, transforming Responses API
|
| 31 |
+
streaming events (output.text.delta, response.completed, etc.) to Interactions
|
| 32 |
+
API streaming events (content.delta, interaction.complete, etc.).
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
def __init__(
|
| 36 |
+
self,
|
| 37 |
+
model: str,
|
| 38 |
+
litellm_custom_stream_wrapper: BaseResponsesAPIStreamingIterator,
|
| 39 |
+
request_input: Optional[InteractionInput],
|
| 40 |
+
optional_params: InteractionsAPIOptionalRequestParams,
|
| 41 |
+
custom_llm_provider: Optional[str] = None,
|
| 42 |
+
litellm_metadata: Optional[Dict[str, Any]] = None,
|
| 43 |
+
):
|
| 44 |
+
self.model = model
|
| 45 |
+
self.responses_stream_iterator = litellm_custom_stream_wrapper
|
| 46 |
+
self.request_input = request_input
|
| 47 |
+
self.optional_params = optional_params
|
| 48 |
+
self.custom_llm_provider = custom_llm_provider
|
| 49 |
+
self.litellm_metadata = litellm_metadata or {}
|
| 50 |
+
self.finished = False
|
| 51 |
+
self.collected_text = ""
|
| 52 |
+
self.sent_interaction_start = False
|
| 53 |
+
self.sent_content_start = False
|
| 54 |
+
|
| 55 |
+
def _transform_responses_chunk_to_interactions_chunk(
|
| 56 |
+
self,
|
| 57 |
+
responses_chunk: ResponsesAPIStreamingResponse,
|
| 58 |
+
) -> Optional[InteractionsAPIStreamingResponse]:
|
| 59 |
+
"""
|
| 60 |
+
Transform a Responses API streaming chunk to an Interactions API streaming chunk.
|
| 61 |
+
|
| 62 |
+
Responses API events:
|
| 63 |
+
- output.text.delta -> content.delta
|
| 64 |
+
- response.completed -> interaction.complete
|
| 65 |
+
|
| 66 |
+
Interactions API events:
|
| 67 |
+
- interaction.start
|
| 68 |
+
- content.start
|
| 69 |
+
- content.delta
|
| 70 |
+
- content.stop
|
| 71 |
+
- interaction.complete
|
| 72 |
+
"""
|
| 73 |
+
if not responses_chunk:
|
| 74 |
+
return None
|
| 75 |
+
|
| 76 |
+
# Handle OutputTextDeltaEvent -> content.delta
|
| 77 |
+
if isinstance(responses_chunk, OutputTextDeltaEvent):
|
| 78 |
+
delta_text = (
|
| 79 |
+
responses_chunk.delta if isinstance(responses_chunk.delta, str) else ""
|
| 80 |
+
)
|
| 81 |
+
self.collected_text += delta_text
|
| 82 |
+
|
| 83 |
+
# Send interaction.start if not sent
|
| 84 |
+
if not self.sent_interaction_start:
|
| 85 |
+
self.sent_interaction_start = True
|
| 86 |
+
return InteractionsAPIStreamingResponse(
|
| 87 |
+
event_type="interaction.start",
|
| 88 |
+
id=getattr(responses_chunk, "item_id", None)
|
| 89 |
+
or f"interaction_{id(self)}",
|
| 90 |
+
object="interaction",
|
| 91 |
+
status="in_progress",
|
| 92 |
+
model=self.model,
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
# Send content.start if not sent
|
| 96 |
+
if not self.sent_content_start:
|
| 97 |
+
self.sent_content_start = True
|
| 98 |
+
return InteractionsAPIStreamingResponse(
|
| 99 |
+
event_type="content.start",
|
| 100 |
+
id=getattr(responses_chunk, "item_id", None),
|
| 101 |
+
object="content",
|
| 102 |
+
delta={"type": "text", "text": ""},
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
# Send content.delta
|
| 106 |
+
return InteractionsAPIStreamingResponse(
|
| 107 |
+
event_type="content.delta",
|
| 108 |
+
id=getattr(responses_chunk, "item_id", None),
|
| 109 |
+
object="content",
|
| 110 |
+
delta={"text": delta_text},
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
# Handle ResponseCreatedEvent or ResponseInProgressEvent -> interaction.start
|
| 114 |
+
if isinstance(responses_chunk, (ResponseCreatedEvent, ResponseInProgressEvent)):
|
| 115 |
+
if not self.sent_interaction_start:
|
| 116 |
+
self.sent_interaction_start = True
|
| 117 |
+
response_id = (
|
| 118 |
+
getattr(responses_chunk.response, "id", None)
|
| 119 |
+
if hasattr(responses_chunk, "response")
|
| 120 |
+
else None
|
| 121 |
+
)
|
| 122 |
+
return InteractionsAPIStreamingResponse(
|
| 123 |
+
event_type="interaction.start",
|
| 124 |
+
id=response_id or f"interaction_{id(self)}",
|
| 125 |
+
object="interaction",
|
| 126 |
+
status="in_progress",
|
| 127 |
+
model=self.model,
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
# Handle ResponseCompletedEvent -> interaction.complete
|
| 131 |
+
if isinstance(responses_chunk, ResponseCompletedEvent):
|
| 132 |
+
self.finished = True
|
| 133 |
+
response = responses_chunk.response
|
| 134 |
+
|
| 135 |
+
# Send content.stop first if content was started
|
| 136 |
+
if self.sent_content_start:
|
| 137 |
+
# Note: We'll send this in the iterator, not here
|
| 138 |
+
pass
|
| 139 |
+
|
| 140 |
+
# Send interaction.complete
|
| 141 |
+
return InteractionsAPIStreamingResponse(
|
| 142 |
+
event_type="interaction.complete",
|
| 143 |
+
id=getattr(response, "id", None) or f"interaction_{id(self)}",
|
| 144 |
+
object="interaction",
|
| 145 |
+
status="completed",
|
| 146 |
+
model=self.model,
|
| 147 |
+
outputs=[
|
| 148 |
+
{
|
| 149 |
+
"type": "text",
|
| 150 |
+
"text": self.collected_text,
|
| 151 |
+
}
|
| 152 |
+
],
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
# For other event types, return None (skip)
|
| 156 |
+
return None
|
| 157 |
+
|
| 158 |
+
def __iter__(self) -> Iterator[InteractionsAPIStreamingResponse]:
|
| 159 |
+
"""Sync iterator implementation."""
|
| 160 |
+
return self
|
| 161 |
+
|
| 162 |
+
def __next__(self) -> InteractionsAPIStreamingResponse:
|
| 163 |
+
"""Get next chunk in sync mode."""
|
| 164 |
+
if self.finished:
|
| 165 |
+
raise StopIteration
|
| 166 |
+
|
| 167 |
+
# Check if we have a pending interaction.complete to send
|
| 168 |
+
if hasattr(self, "_pending_interaction_complete"):
|
| 169 |
+
pending: InteractionsAPIStreamingResponse = getattr(
|
| 170 |
+
self, "_pending_interaction_complete"
|
| 171 |
+
)
|
| 172 |
+
delattr(self, "_pending_interaction_complete")
|
| 173 |
+
return pending
|
| 174 |
+
|
| 175 |
+
# Use a loop instead of recursion to avoid stack overflow
|
| 176 |
+
sync_iterator = cast(
|
| 177 |
+
SyncResponsesAPIStreamingIterator, self.responses_stream_iterator
|
| 178 |
+
)
|
| 179 |
+
while True:
|
| 180 |
+
try:
|
| 181 |
+
# Get next chunk from responses API stream
|
| 182 |
+
chunk = next(sync_iterator)
|
| 183 |
+
|
| 184 |
+
# Transform chunk (chunk is already a ResponsesAPIStreamingResponse)
|
| 185 |
+
transformed = self._transform_responses_chunk_to_interactions_chunk(
|
| 186 |
+
chunk
|
| 187 |
+
)
|
| 188 |
+
|
| 189 |
+
if transformed:
|
| 190 |
+
# If we finished and content was started, send content.stop before interaction.complete
|
| 191 |
+
if (
|
| 192 |
+
self.finished
|
| 193 |
+
and self.sent_content_start
|
| 194 |
+
and transformed.event_type == "interaction.complete"
|
| 195 |
+
):
|
| 196 |
+
# Send content.stop first
|
| 197 |
+
content_stop = InteractionsAPIStreamingResponse(
|
| 198 |
+
event_type="content.stop",
|
| 199 |
+
id=transformed.id,
|
| 200 |
+
object="content",
|
| 201 |
+
delta={"type": "text", "text": self.collected_text},
|
| 202 |
+
)
|
| 203 |
+
# Store the interaction.complete to send next
|
| 204 |
+
self._pending_interaction_complete = transformed
|
| 205 |
+
return content_stop
|
| 206 |
+
return transformed
|
| 207 |
+
|
| 208 |
+
# If no transformation, continue to next chunk (loop continues)
|
| 209 |
+
|
| 210 |
+
except StopIteration:
|
| 211 |
+
self.finished = True
|
| 212 |
+
|
| 213 |
+
# Send final events if needed
|
| 214 |
+
if self.sent_content_start:
|
| 215 |
+
return InteractionsAPIStreamingResponse(
|
| 216 |
+
event_type="content.stop",
|
| 217 |
+
object="content",
|
| 218 |
+
delta={"type": "text", "text": self.collected_text},
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
raise StopIteration
|
| 222 |
+
|
| 223 |
+
def __aiter__(self) -> AsyncIterator[InteractionsAPIStreamingResponse]:
|
| 224 |
+
"""Async iterator implementation."""
|
| 225 |
+
return self
|
| 226 |
+
|
| 227 |
+
async def __anext__(self) -> InteractionsAPIStreamingResponse:
|
| 228 |
+
"""Get next chunk in async mode."""
|
| 229 |
+
if self.finished:
|
| 230 |
+
raise StopAsyncIteration
|
| 231 |
+
|
| 232 |
+
# Check if we have a pending interaction.complete to send
|
| 233 |
+
if hasattr(self, "_pending_interaction_complete"):
|
| 234 |
+
pending: InteractionsAPIStreamingResponse = getattr(
|
| 235 |
+
self, "_pending_interaction_complete"
|
| 236 |
+
)
|
| 237 |
+
delattr(self, "_pending_interaction_complete")
|
| 238 |
+
return pending
|
| 239 |
+
|
| 240 |
+
# Use a loop instead of recursion to avoid stack overflow
|
| 241 |
+
async_iterator = cast(
|
| 242 |
+
ResponsesAPIStreamingIterator, self.responses_stream_iterator
|
| 243 |
+
)
|
| 244 |
+
while True:
|
| 245 |
+
try:
|
| 246 |
+
# Get next chunk from responses API stream
|
| 247 |
+
chunk = await async_iterator.__anext__()
|
| 248 |
+
|
| 249 |
+
# Transform chunk (chunk is already a ResponsesAPIStreamingResponse)
|
| 250 |
+
transformed = self._transform_responses_chunk_to_interactions_chunk(
|
| 251 |
+
chunk
|
| 252 |
+
)
|
| 253 |
+
|
| 254 |
+
if transformed:
|
| 255 |
+
# If we finished and content was started, send content.stop before interaction.complete
|
| 256 |
+
if (
|
| 257 |
+
self.finished
|
| 258 |
+
and self.sent_content_start
|
| 259 |
+
and transformed.event_type == "interaction.complete"
|
| 260 |
+
):
|
| 261 |
+
# Send content.stop first
|
| 262 |
+
content_stop = InteractionsAPIStreamingResponse(
|
| 263 |
+
event_type="content.stop",
|
| 264 |
+
id=transformed.id,
|
| 265 |
+
object="content",
|
| 266 |
+
delta={"type": "text", "text": self.collected_text},
|
| 267 |
+
)
|
| 268 |
+
# Store the interaction.complete to send next
|
| 269 |
+
self._pending_interaction_complete = transformed
|
| 270 |
+
return content_stop
|
| 271 |
+
return transformed
|
| 272 |
+
|
| 273 |
+
# If no transformation, continue to next chunk (loop continues)
|
| 274 |
+
|
| 275 |
+
except StopAsyncIteration:
|
| 276 |
+
self.finished = True
|
| 277 |
+
|
| 278 |
+
# Send final events if needed
|
| 279 |
+
if self.sent_content_start:
|
| 280 |
+
return InteractionsAPIStreamingResponse(
|
| 281 |
+
event_type="content.stop",
|
| 282 |
+
object="content",
|
| 283 |
+
delta={"type": "text", "text": self.collected_text},
|
| 284 |
+
)
|
| 285 |
+
|
| 286 |
+
raise StopAsyncIteration
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/transformation.py
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation utilities for bridging Interactions API to Responses API.
|
| 3 |
+
|
| 4 |
+
This module handles transforming between:
|
| 5 |
+
- Interactions API format (Google's format with Turn[], system_instruction, etc.)
|
| 6 |
+
- Responses API format (OpenAI's format with input[], instructions, etc.)
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from typing import Any, Dict, List, Optional, cast
|
| 10 |
+
|
| 11 |
+
from litellm.types.interactions import (
|
| 12 |
+
InteractionInput,
|
| 13 |
+
InteractionsAPIOptionalRequestParams,
|
| 14 |
+
InteractionsAPIResponse,
|
| 15 |
+
Turn,
|
| 16 |
+
)
|
| 17 |
+
from litellm.types.llms.openai import (
|
| 18 |
+
ResponseInputParam,
|
| 19 |
+
ResponsesAPIResponse,
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class LiteLLMResponsesInteractionsConfig:
|
| 24 |
+
"""Configuration class for transforming between Interactions API and Responses API."""
|
| 25 |
+
|
| 26 |
+
@staticmethod
|
| 27 |
+
def transform_interactions_request_to_responses_request(
|
| 28 |
+
model: str,
|
| 29 |
+
input: Optional[InteractionInput],
|
| 30 |
+
optional_params: InteractionsAPIOptionalRequestParams,
|
| 31 |
+
**kwargs,
|
| 32 |
+
) -> Dict[str, Any]:
|
| 33 |
+
"""
|
| 34 |
+
Transform an Interactions API request to a Responses API request.
|
| 35 |
+
|
| 36 |
+
Key transformations:
|
| 37 |
+
- system_instruction -> instructions
|
| 38 |
+
- input (string | Turn[]) -> input (ResponseInputParam)
|
| 39 |
+
- tools -> tools (similar format)
|
| 40 |
+
- generation_config -> temperature, top_p, etc.
|
| 41 |
+
"""
|
| 42 |
+
responses_request: Dict[str, Any] = {
|
| 43 |
+
"model": model,
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
# Transform input
|
| 47 |
+
if input is not None:
|
| 48 |
+
responses_request[
|
| 49 |
+
"input"
|
| 50 |
+
] = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
|
| 51 |
+
input
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
# Transform system_instruction -> instructions
|
| 55 |
+
if optional_params.get("system_instruction"):
|
| 56 |
+
responses_request["instructions"] = optional_params["system_instruction"]
|
| 57 |
+
|
| 58 |
+
# Transform tools (similar format, pass through for now)
|
| 59 |
+
if optional_params.get("tools"):
|
| 60 |
+
responses_request["tools"] = optional_params["tools"]
|
| 61 |
+
|
| 62 |
+
# Transform generation_config to temperature, top_p, etc.
|
| 63 |
+
generation_config = optional_params.get("generation_config")
|
| 64 |
+
if generation_config:
|
| 65 |
+
if isinstance(generation_config, dict):
|
| 66 |
+
if "temperature" in generation_config:
|
| 67 |
+
responses_request["temperature"] = generation_config["temperature"]
|
| 68 |
+
if "top_p" in generation_config:
|
| 69 |
+
responses_request["top_p"] = generation_config["top_p"]
|
| 70 |
+
if "top_k" in generation_config:
|
| 71 |
+
# Responses API doesn't have top_k, skip it
|
| 72 |
+
pass
|
| 73 |
+
if "max_output_tokens" in generation_config:
|
| 74 |
+
responses_request["max_output_tokens"] = generation_config[
|
| 75 |
+
"max_output_tokens"
|
| 76 |
+
]
|
| 77 |
+
|
| 78 |
+
# Pass through other optional params that match
|
| 79 |
+
passthrough_params = ["stream", "store", "metadata", "user"]
|
| 80 |
+
for param in passthrough_params:
|
| 81 |
+
if param in optional_params and optional_params[param] is not None:
|
| 82 |
+
responses_request[param] = optional_params[param]
|
| 83 |
+
|
| 84 |
+
# Add any extra kwargs
|
| 85 |
+
responses_request.update(kwargs)
|
| 86 |
+
|
| 87 |
+
return responses_request
|
| 88 |
+
|
| 89 |
+
@staticmethod
|
| 90 |
+
def _transform_interactions_input_to_responses_input(
|
| 91 |
+
input: InteractionInput,
|
| 92 |
+
) -> ResponseInputParam:
|
| 93 |
+
"""
|
| 94 |
+
Transform Interactions API input to Responses API input format.
|
| 95 |
+
|
| 96 |
+
Interactions API input can be:
|
| 97 |
+
- string: "Hello"
|
| 98 |
+
- Turn[]: [{"role": "user", "content": [...]}]
|
| 99 |
+
- Content object
|
| 100 |
+
|
| 101 |
+
Responses API input is:
|
| 102 |
+
- string: "Hello"
|
| 103 |
+
- Message[]: [{"role": "user", "content": [...]}]
|
| 104 |
+
"""
|
| 105 |
+
if isinstance(input, str):
|
| 106 |
+
# ResponseInputParam accepts str
|
| 107 |
+
return cast(ResponseInputParam, input)
|
| 108 |
+
|
| 109 |
+
if isinstance(input, list):
|
| 110 |
+
# Turn[] format - convert to Responses API Message[] format
|
| 111 |
+
messages = []
|
| 112 |
+
for turn in input:
|
| 113 |
+
if isinstance(turn, dict):
|
| 114 |
+
role = turn.get("role", "user")
|
| 115 |
+
content = turn.get("content", [])
|
| 116 |
+
|
| 117 |
+
# Transform content array
|
| 118 |
+
transformed_content = (
|
| 119 |
+
LiteLLMResponsesInteractionsConfig._transform_content_array(
|
| 120 |
+
content
|
| 121 |
+
)
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
messages.append(
|
| 125 |
+
{
|
| 126 |
+
"role": role,
|
| 127 |
+
"content": transformed_content,
|
| 128 |
+
}
|
| 129 |
+
)
|
| 130 |
+
elif isinstance(turn, Turn):
|
| 131 |
+
# Pydantic model
|
| 132 |
+
role = turn.role if hasattr(turn, "role") else "user"
|
| 133 |
+
content = turn.content if hasattr(turn, "content") else []
|
| 134 |
+
|
| 135 |
+
# Ensure content is a list for _transform_content_array
|
| 136 |
+
# Cast to List[Any] to handle various content types
|
| 137 |
+
if isinstance(content, list):
|
| 138 |
+
content_list: List[Any] = list(content)
|
| 139 |
+
elif content is not None:
|
| 140 |
+
content_list = [content]
|
| 141 |
+
else:
|
| 142 |
+
content_list = []
|
| 143 |
+
|
| 144 |
+
transformed_content = (
|
| 145 |
+
LiteLLMResponsesInteractionsConfig._transform_content_array(
|
| 146 |
+
content_list
|
| 147 |
+
)
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
messages.append(
|
| 151 |
+
{
|
| 152 |
+
"role": role,
|
| 153 |
+
"content": transformed_content,
|
| 154 |
+
}
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
+
return cast(ResponseInputParam, messages)
|
| 158 |
+
|
| 159 |
+
# Single content object - wrap in message
|
| 160 |
+
if isinstance(input, dict):
|
| 161 |
+
return cast(
|
| 162 |
+
ResponseInputParam,
|
| 163 |
+
[
|
| 164 |
+
{
|
| 165 |
+
"role": "user",
|
| 166 |
+
"content": LiteLLMResponsesInteractionsConfig._transform_content_array(
|
| 167 |
+
input.get("content", [])
|
| 168 |
+
if isinstance(input.get("content"), list)
|
| 169 |
+
else [input]
|
| 170 |
+
),
|
| 171 |
+
}
|
| 172 |
+
],
|
| 173 |
+
)
|
| 174 |
+
|
| 175 |
+
# Fallback: convert to string
|
| 176 |
+
return cast(ResponseInputParam, str(input))
|
| 177 |
+
|
| 178 |
+
@staticmethod
|
| 179 |
+
def _transform_content_array(content: List[Any]) -> List[Dict[str, Any]]:
|
| 180 |
+
"""Transform Interactions API content array to Responses API format."""
|
| 181 |
+
if not isinstance(content, list):
|
| 182 |
+
# Single content item - wrap in array
|
| 183 |
+
content = [content]
|
| 184 |
+
|
| 185 |
+
transformed: List[Dict[str, Any]] = []
|
| 186 |
+
for item in content:
|
| 187 |
+
if isinstance(item, dict):
|
| 188 |
+
# Already in dict format, pass through
|
| 189 |
+
transformed.append(item)
|
| 190 |
+
elif isinstance(item, str):
|
| 191 |
+
# Plain string - wrap in text format
|
| 192 |
+
transformed.append({"type": "text", "text": item})
|
| 193 |
+
else:
|
| 194 |
+
# Pydantic model or other - convert to dict
|
| 195 |
+
if hasattr(item, "model_dump"):
|
| 196 |
+
dumped = item.model_dump()
|
| 197 |
+
if isinstance(dumped, dict):
|
| 198 |
+
transformed.append(dumped)
|
| 199 |
+
else:
|
| 200 |
+
# Fallback: wrap in text format
|
| 201 |
+
transformed.append({"type": "text", "text": str(dumped)})
|
| 202 |
+
elif hasattr(item, "dict"):
|
| 203 |
+
dumped = item.dict()
|
| 204 |
+
if isinstance(dumped, dict):
|
| 205 |
+
transformed.append(dumped)
|
| 206 |
+
else:
|
| 207 |
+
# Fallback: wrap in text format
|
| 208 |
+
transformed.append({"type": "text", "text": str(dumped)})
|
| 209 |
+
else:
|
| 210 |
+
# Fallback: wrap in text format
|
| 211 |
+
transformed.append({"type": "text", "text": str(item)})
|
| 212 |
+
|
| 213 |
+
return transformed
|
| 214 |
+
|
| 215 |
+
@staticmethod
|
| 216 |
+
def transform_responses_response_to_interactions_response(
|
| 217 |
+
responses_response: ResponsesAPIResponse,
|
| 218 |
+
model: Optional[str] = None,
|
| 219 |
+
) -> InteractionsAPIResponse:
|
| 220 |
+
"""
|
| 221 |
+
Transform a Responses API response to an Interactions API response.
|
| 222 |
+
|
| 223 |
+
Key transformations:
|
| 224 |
+
- Extract text from output[].content[].text
|
| 225 |
+
- Convert created_at (int) to created (ISO string)
|
| 226 |
+
- Map status
|
| 227 |
+
- Extract usage
|
| 228 |
+
"""
|
| 229 |
+
# Extract text from outputs
|
| 230 |
+
outputs = []
|
| 231 |
+
if hasattr(responses_response, "output") and responses_response.output:
|
| 232 |
+
for output_item in responses_response.output:
|
| 233 |
+
# Use getattr with None default to safely access content
|
| 234 |
+
content = getattr(output_item, "content", None)
|
| 235 |
+
if content is not None:
|
| 236 |
+
content_items = content if isinstance(content, list) else [content]
|
| 237 |
+
for content_item in content_items:
|
| 238 |
+
# Check if content_item has text attribute
|
| 239 |
+
text = getattr(content_item, "text", None)
|
| 240 |
+
if text is not None:
|
| 241 |
+
outputs.append(
|
| 242 |
+
{
|
| 243 |
+
"type": "text",
|
| 244 |
+
"text": text,
|
| 245 |
+
}
|
| 246 |
+
)
|
| 247 |
+
elif (
|
| 248 |
+
isinstance(content_item, dict)
|
| 249 |
+
and content_item.get("type") == "text"
|
| 250 |
+
):
|
| 251 |
+
outputs.append(content_item)
|
| 252 |
+
|
| 253 |
+
# Convert created_at to ISO string
|
| 254 |
+
created_at = getattr(responses_response, "created_at", None)
|
| 255 |
+
if isinstance(created_at, int):
|
| 256 |
+
from datetime import datetime
|
| 257 |
+
|
| 258 |
+
created = datetime.fromtimestamp(created_at).isoformat()
|
| 259 |
+
elif created_at is not None and hasattr(created_at, "isoformat"):
|
| 260 |
+
created = created_at.isoformat()
|
| 261 |
+
else:
|
| 262 |
+
created = None
|
| 263 |
+
|
| 264 |
+
# Map status
|
| 265 |
+
status = getattr(responses_response, "status", "completed")
|
| 266 |
+
if status == "completed":
|
| 267 |
+
interactions_status = "completed"
|
| 268 |
+
elif status == "in_progress":
|
| 269 |
+
interactions_status = "in_progress"
|
| 270 |
+
else:
|
| 271 |
+
interactions_status = status
|
| 272 |
+
|
| 273 |
+
# Build interactions response
|
| 274 |
+
interactions_response_dict: Dict[str, Any] = {
|
| 275 |
+
"id": getattr(responses_response, "id", ""),
|
| 276 |
+
"object": "interaction",
|
| 277 |
+
"status": interactions_status,
|
| 278 |
+
"outputs": outputs,
|
| 279 |
+
"model": model or getattr(responses_response, "model", ""),
|
| 280 |
+
"created": created,
|
| 281 |
+
}
|
| 282 |
+
|
| 283 |
+
# Add usage if available
|
| 284 |
+
# Map Responses API usage (input_tokens, output_tokens) to Interactions API spec format
|
| 285 |
+
# (total_input_tokens, total_output_tokens)
|
| 286 |
+
usage = getattr(responses_response, "usage", None)
|
| 287 |
+
if usage:
|
| 288 |
+
interactions_response_dict["usage"] = {
|
| 289 |
+
"total_input_tokens": getattr(usage, "input_tokens", 0),
|
| 290 |
+
"total_output_tokens": getattr(usage, "output_tokens", 0),
|
| 291 |
+
}
|
| 292 |
+
|
| 293 |
+
# Add role
|
| 294 |
+
interactions_response_dict["role"] = "model"
|
| 295 |
+
|
| 296 |
+
# Add updated (same as created for now)
|
| 297 |
+
interactions_response_dict["updated"] = created
|
| 298 |
+
|
| 299 |
+
return InteractionsAPIResponse(**interactions_response_dict)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/main.py
ADDED
|
@@ -0,0 +1,644 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
LiteLLM Interactions API - Main Module
|
| 3 |
+
|
| 4 |
+
Per OpenAPI spec (https://ai.google.dev/static/api/interactions.openapi.json):
|
| 5 |
+
- Create interaction: POST /{api_version}/interactions
|
| 6 |
+
- Get interaction: GET /{api_version}/interactions/{interaction_id}
|
| 7 |
+
- Delete interaction: DELETE /{api_version}/interactions/{interaction_id}
|
| 8 |
+
|
| 9 |
+
Usage:
|
| 10 |
+
import litellm
|
| 11 |
+
|
| 12 |
+
# Create an interaction with a model
|
| 13 |
+
response = litellm.interactions.create(
|
| 14 |
+
model="gemini-2.5-flash",
|
| 15 |
+
input="Hello, how are you?"
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
# Create an interaction with an agent
|
| 19 |
+
response = litellm.interactions.create(
|
| 20 |
+
agent="deep-research-pro-preview-12-2025",
|
| 21 |
+
input="Research the current state of cancer research"
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
# Async version
|
| 25 |
+
response = await litellm.interactions.acreate(...)
|
| 26 |
+
|
| 27 |
+
# Get an interaction
|
| 28 |
+
response = litellm.interactions.get(interaction_id="...")
|
| 29 |
+
|
| 30 |
+
# Delete an interaction
|
| 31 |
+
result = litellm.interactions.delete(interaction_id="...")
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
import asyncio
|
| 35 |
+
import contextvars
|
| 36 |
+
from functools import partial
|
| 37 |
+
from typing import Any, AsyncIterator, Coroutine, Dict, Iterator, List, Optional, Union
|
| 38 |
+
|
| 39 |
+
import httpx
|
| 40 |
+
|
| 41 |
+
import litellm
|
| 42 |
+
from litellm.interactions.http_handler import interactions_http_handler
|
| 43 |
+
from litellm.interactions.utils import (
|
| 44 |
+
InteractionsAPIRequestUtils,
|
| 45 |
+
get_provider_interactions_api_config,
|
| 46 |
+
)
|
| 47 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
| 48 |
+
from litellm.types.interactions import (
|
| 49 |
+
CancelInteractionResult,
|
| 50 |
+
DeleteInteractionResult,
|
| 51 |
+
InteractionInput,
|
| 52 |
+
InteractionsAPIResponse,
|
| 53 |
+
InteractionsAPIStreamingResponse,
|
| 54 |
+
InteractionTool,
|
| 55 |
+
)
|
| 56 |
+
from litellm.types.router import GenericLiteLLMParams
|
| 57 |
+
from litellm.utils import client
|
| 58 |
+
|
| 59 |
+
# ============================================================
|
| 60 |
+
# SDK Methods - CREATE INTERACTION
|
| 61 |
+
# ============================================================
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@client
|
| 65 |
+
async def acreate(
|
| 66 |
+
# Model or Agent (one required per OpenAPI spec)
|
| 67 |
+
model: Optional[str] = None,
|
| 68 |
+
agent: Optional[str] = None,
|
| 69 |
+
# Input (required)
|
| 70 |
+
input: Optional[InteractionInput] = None,
|
| 71 |
+
# Tools (for model interactions)
|
| 72 |
+
tools: Optional[List[InteractionTool]] = None,
|
| 73 |
+
# System instruction
|
| 74 |
+
system_instruction: Optional[str] = None,
|
| 75 |
+
# Generation config
|
| 76 |
+
generation_config: Optional[Dict[str, Any]] = None,
|
| 77 |
+
# Streaming
|
| 78 |
+
stream: Optional[bool] = None,
|
| 79 |
+
# Storage
|
| 80 |
+
store: Optional[bool] = None,
|
| 81 |
+
# Background execution
|
| 82 |
+
background: Optional[bool] = None,
|
| 83 |
+
# Response format
|
| 84 |
+
response_modalities: Optional[List[str]] = None,
|
| 85 |
+
response_format: Optional[Dict[str, Any]] = None,
|
| 86 |
+
response_mime_type: Optional[str] = None,
|
| 87 |
+
# Continuation
|
| 88 |
+
previous_interaction_id: Optional[str] = None,
|
| 89 |
+
# Extra params
|
| 90 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 91 |
+
extra_body: Optional[Dict[str, Any]] = None,
|
| 92 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 93 |
+
# LiteLLM params
|
| 94 |
+
custom_llm_provider: Optional[str] = None,
|
| 95 |
+
**kwargs,
|
| 96 |
+
) -> Union[InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]]:
|
| 97 |
+
"""
|
| 98 |
+
Async: Create a new interaction using Google's Interactions API.
|
| 99 |
+
|
| 100 |
+
Per OpenAPI spec, provide either `model` or `agent`.
|
| 101 |
+
|
| 102 |
+
Args:
|
| 103 |
+
model: The model to use (e.g., "gemini-2.5-flash")
|
| 104 |
+
agent: The agent to use (e.g., "deep-research-pro-preview-12-2025")
|
| 105 |
+
input: The input content (string, content object, or list)
|
| 106 |
+
tools: Tools available for the model
|
| 107 |
+
system_instruction: System instruction for the interaction
|
| 108 |
+
generation_config: Generation configuration
|
| 109 |
+
stream: Whether to stream the response
|
| 110 |
+
store: Whether to store the response for later retrieval
|
| 111 |
+
background: Whether to run in background
|
| 112 |
+
response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO)
|
| 113 |
+
response_format: JSON schema for response format
|
| 114 |
+
response_mime_type: MIME type of the response
|
| 115 |
+
previous_interaction_id: ID of previous interaction for continuation
|
| 116 |
+
extra_headers: Additional headers
|
| 117 |
+
extra_body: Additional body parameters
|
| 118 |
+
timeout: Request timeout
|
| 119 |
+
custom_llm_provider: Override the LLM provider
|
| 120 |
+
|
| 121 |
+
Returns:
|
| 122 |
+
InteractionsAPIResponse or async iterator for streaming
|
| 123 |
+
"""
|
| 124 |
+
local_vars = locals()
|
| 125 |
+
try:
|
| 126 |
+
loop = asyncio.get_event_loop()
|
| 127 |
+
kwargs["acreate_interaction"] = True
|
| 128 |
+
|
| 129 |
+
if custom_llm_provider is None and model:
|
| 130 |
+
_, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
| 131 |
+
model=model, api_base=kwargs.get("api_base", None)
|
| 132 |
+
)
|
| 133 |
+
elif custom_llm_provider is None:
|
| 134 |
+
custom_llm_provider = "gemini"
|
| 135 |
+
|
| 136 |
+
func = partial(
|
| 137 |
+
create,
|
| 138 |
+
model=model,
|
| 139 |
+
agent=agent,
|
| 140 |
+
input=input,
|
| 141 |
+
tools=tools,
|
| 142 |
+
system_instruction=system_instruction,
|
| 143 |
+
generation_config=generation_config,
|
| 144 |
+
stream=stream,
|
| 145 |
+
store=store,
|
| 146 |
+
background=background,
|
| 147 |
+
response_modalities=response_modalities,
|
| 148 |
+
response_format=response_format,
|
| 149 |
+
response_mime_type=response_mime_type,
|
| 150 |
+
previous_interaction_id=previous_interaction_id,
|
| 151 |
+
extra_headers=extra_headers,
|
| 152 |
+
extra_body=extra_body,
|
| 153 |
+
timeout=timeout,
|
| 154 |
+
custom_llm_provider=custom_llm_provider,
|
| 155 |
+
**kwargs,
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
ctx = contextvars.copy_context()
|
| 159 |
+
func_with_context = partial(ctx.run, func)
|
| 160 |
+
init_response = await loop.run_in_executor(None, func_with_context)
|
| 161 |
+
|
| 162 |
+
if asyncio.iscoroutine(init_response):
|
| 163 |
+
response = await init_response
|
| 164 |
+
else:
|
| 165 |
+
response = init_response
|
| 166 |
+
|
| 167 |
+
return response # type: ignore
|
| 168 |
+
except Exception as e:
|
| 169 |
+
raise litellm.exception_type(
|
| 170 |
+
model=model,
|
| 171 |
+
custom_llm_provider=custom_llm_provider,
|
| 172 |
+
original_exception=e,
|
| 173 |
+
completion_kwargs=local_vars,
|
| 174 |
+
extra_kwargs=kwargs,
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
@client
|
| 179 |
+
def create(
|
| 180 |
+
# Model or Agent (one required per OpenAPI spec)
|
| 181 |
+
model: Optional[str] = None,
|
| 182 |
+
agent: Optional[str] = None,
|
| 183 |
+
# Input (required)
|
| 184 |
+
input: Optional[InteractionInput] = None,
|
| 185 |
+
# Tools (for model interactions)
|
| 186 |
+
tools: Optional[List[InteractionTool]] = None,
|
| 187 |
+
# System instruction
|
| 188 |
+
system_instruction: Optional[str] = None,
|
| 189 |
+
# Generation config
|
| 190 |
+
generation_config: Optional[Dict[str, Any]] = None,
|
| 191 |
+
# Streaming
|
| 192 |
+
stream: Optional[bool] = None,
|
| 193 |
+
# Storage
|
| 194 |
+
store: Optional[bool] = None,
|
| 195 |
+
# Background execution
|
| 196 |
+
background: Optional[bool] = None,
|
| 197 |
+
# Response format
|
| 198 |
+
response_modalities: Optional[List[str]] = None,
|
| 199 |
+
response_format: Optional[Dict[str, Any]] = None,
|
| 200 |
+
response_mime_type: Optional[str] = None,
|
| 201 |
+
# Continuation
|
| 202 |
+
previous_interaction_id: Optional[str] = None,
|
| 203 |
+
# Extra params
|
| 204 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 205 |
+
extra_body: Optional[Dict[str, Any]] = None,
|
| 206 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 207 |
+
# LiteLLM params
|
| 208 |
+
custom_llm_provider: Optional[str] = None,
|
| 209 |
+
**kwargs,
|
| 210 |
+
) -> Union[
|
| 211 |
+
InteractionsAPIResponse,
|
| 212 |
+
Iterator[InteractionsAPIStreamingResponse],
|
| 213 |
+
Coroutine[
|
| 214 |
+
Any,
|
| 215 |
+
Any,
|
| 216 |
+
Union[InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]],
|
| 217 |
+
],
|
| 218 |
+
]:
|
| 219 |
+
"""
|
| 220 |
+
Sync: Create a new interaction using Google's Interactions API.
|
| 221 |
+
|
| 222 |
+
Per OpenAPI spec, provide either `model` or `agent`.
|
| 223 |
+
|
| 224 |
+
Args:
|
| 225 |
+
model: The model to use (e.g., "gemini-2.5-flash")
|
| 226 |
+
agent: The agent to use (e.g., "deep-research-pro-preview-12-2025")
|
| 227 |
+
input: The input content (string, content object, or list)
|
| 228 |
+
tools: Tools available for the model
|
| 229 |
+
system_instruction: System instruction for the interaction
|
| 230 |
+
generation_config: Generation configuration
|
| 231 |
+
stream: Whether to stream the response
|
| 232 |
+
store: Whether to store the response for later retrieval
|
| 233 |
+
background: Whether to run in background
|
| 234 |
+
response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO)
|
| 235 |
+
response_format: JSON schema for response format
|
| 236 |
+
response_mime_type: MIME type of the response
|
| 237 |
+
previous_interaction_id: ID of previous interaction for continuation
|
| 238 |
+
extra_headers: Additional headers
|
| 239 |
+
extra_body: Additional body parameters
|
| 240 |
+
timeout: Request timeout
|
| 241 |
+
custom_llm_provider: Override the LLM provider
|
| 242 |
+
|
| 243 |
+
Returns:
|
| 244 |
+
InteractionsAPIResponse or iterator for streaming
|
| 245 |
+
"""
|
| 246 |
+
local_vars = locals()
|
| 247 |
+
|
| 248 |
+
try:
|
| 249 |
+
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
|
| 250 |
+
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
|
| 251 |
+
_is_async = kwargs.pop("acreate_interaction", False) is True
|
| 252 |
+
|
| 253 |
+
litellm_params = GenericLiteLLMParams(**kwargs)
|
| 254 |
+
|
| 255 |
+
if model:
|
| 256 |
+
model, custom_llm_provider, _, _ = litellm.get_llm_provider(
|
| 257 |
+
model=model,
|
| 258 |
+
custom_llm_provider=custom_llm_provider,
|
| 259 |
+
api_base=litellm_params.api_base,
|
| 260 |
+
api_key=litellm_params.api_key,
|
| 261 |
+
)
|
| 262 |
+
else:
|
| 263 |
+
custom_llm_provider = custom_llm_provider or "gemini"
|
| 264 |
+
|
| 265 |
+
interactions_api_config = get_provider_interactions_api_config(
|
| 266 |
+
provider=custom_llm_provider,
|
| 267 |
+
model=model,
|
| 268 |
+
)
|
| 269 |
+
|
| 270 |
+
# Get optional params using utility (similar to responses API pattern)
|
| 271 |
+
local_vars.update(kwargs)
|
| 272 |
+
optional_params = (
|
| 273 |
+
InteractionsAPIRequestUtils.get_requested_interactions_api_optional_params(
|
| 274 |
+
local_vars
|
| 275 |
+
)
|
| 276 |
+
)
|
| 277 |
+
|
| 278 |
+
# Check if this is a bridge provider (litellm_responses) - similar to responses API
|
| 279 |
+
# Either provider is explicitly "litellm_responses" or no config found (bridge to responses)
|
| 280 |
+
if (
|
| 281 |
+
custom_llm_provider == "litellm_responses"
|
| 282 |
+
or interactions_api_config is None
|
| 283 |
+
):
|
| 284 |
+
# Bridge to litellm.responses() for non-native providers
|
| 285 |
+
from litellm.interactions.litellm_responses_transformation.handler import (
|
| 286 |
+
LiteLLMResponsesInteractionsHandler,
|
| 287 |
+
)
|
| 288 |
+
|
| 289 |
+
handler = LiteLLMResponsesInteractionsHandler()
|
| 290 |
+
return handler.interactions_api_handler(
|
| 291 |
+
model=model or "",
|
| 292 |
+
input=input,
|
| 293 |
+
optional_params=optional_params,
|
| 294 |
+
custom_llm_provider=custom_llm_provider,
|
| 295 |
+
_is_async=_is_async,
|
| 296 |
+
stream=stream,
|
| 297 |
+
**kwargs,
|
| 298 |
+
)
|
| 299 |
+
|
| 300 |
+
litellm_logging_obj.update_from_kwargs(
|
| 301 |
+
kwargs=kwargs,
|
| 302 |
+
model=model,
|
| 303 |
+
optional_params=dict(optional_params),
|
| 304 |
+
litellm_params={"litellm_call_id": litellm_call_id},
|
| 305 |
+
custom_llm_provider=custom_llm_provider,
|
| 306 |
+
)
|
| 307 |
+
|
| 308 |
+
response = interactions_http_handler.create_interaction(
|
| 309 |
+
model=model,
|
| 310 |
+
agent=agent,
|
| 311 |
+
input=input,
|
| 312 |
+
interactions_api_config=interactions_api_config,
|
| 313 |
+
optional_params=optional_params,
|
| 314 |
+
custom_llm_provider=custom_llm_provider,
|
| 315 |
+
litellm_params=litellm_params,
|
| 316 |
+
logging_obj=litellm_logging_obj,
|
| 317 |
+
extra_headers=extra_headers,
|
| 318 |
+
extra_body=extra_body,
|
| 319 |
+
timeout=timeout,
|
| 320 |
+
_is_async=_is_async,
|
| 321 |
+
stream=stream,
|
| 322 |
+
)
|
| 323 |
+
|
| 324 |
+
return response
|
| 325 |
+
except Exception as e:
|
| 326 |
+
raise litellm.exception_type(
|
| 327 |
+
model=model,
|
| 328 |
+
custom_llm_provider=custom_llm_provider,
|
| 329 |
+
original_exception=e,
|
| 330 |
+
completion_kwargs=local_vars,
|
| 331 |
+
extra_kwargs=kwargs,
|
| 332 |
+
)
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
# ============================================================
|
| 336 |
+
# SDK Methods - GET INTERACTION
|
| 337 |
+
# ============================================================
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
@client
|
| 341 |
+
async def aget(
|
| 342 |
+
interaction_id: str,
|
| 343 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 344 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 345 |
+
custom_llm_provider: Optional[str] = None,
|
| 346 |
+
**kwargs,
|
| 347 |
+
) -> InteractionsAPIResponse:
|
| 348 |
+
"""Async: Get an interaction by its ID."""
|
| 349 |
+
local_vars = locals()
|
| 350 |
+
try:
|
| 351 |
+
loop = asyncio.get_event_loop()
|
| 352 |
+
kwargs["aget_interaction"] = True
|
| 353 |
+
|
| 354 |
+
func = partial(
|
| 355 |
+
get,
|
| 356 |
+
interaction_id=interaction_id,
|
| 357 |
+
extra_headers=extra_headers,
|
| 358 |
+
timeout=timeout,
|
| 359 |
+
custom_llm_provider=custom_llm_provider or "gemini",
|
| 360 |
+
**kwargs,
|
| 361 |
+
)
|
| 362 |
+
|
| 363 |
+
ctx = contextvars.copy_context()
|
| 364 |
+
func_with_context = partial(ctx.run, func)
|
| 365 |
+
init_response = await loop.run_in_executor(None, func_with_context)
|
| 366 |
+
|
| 367 |
+
if asyncio.iscoroutine(init_response):
|
| 368 |
+
response = await init_response
|
| 369 |
+
else:
|
| 370 |
+
response = init_response
|
| 371 |
+
|
| 372 |
+
return response # type: ignore
|
| 373 |
+
except Exception as e:
|
| 374 |
+
raise litellm.exception_type(
|
| 375 |
+
model=None,
|
| 376 |
+
custom_llm_provider=custom_llm_provider or "gemini",
|
| 377 |
+
original_exception=e,
|
| 378 |
+
completion_kwargs=local_vars,
|
| 379 |
+
extra_kwargs=kwargs,
|
| 380 |
+
)
|
| 381 |
+
|
| 382 |
+
|
| 383 |
+
@client
|
| 384 |
+
def get(
|
| 385 |
+
interaction_id: str,
|
| 386 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 387 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 388 |
+
custom_llm_provider: Optional[str] = None,
|
| 389 |
+
**kwargs,
|
| 390 |
+
) -> Union[InteractionsAPIResponse, Coroutine[Any, Any, InteractionsAPIResponse]]:
|
| 391 |
+
"""Sync: Get an interaction by its ID."""
|
| 392 |
+
local_vars = locals()
|
| 393 |
+
custom_llm_provider = custom_llm_provider or "gemini"
|
| 394 |
+
|
| 395 |
+
try:
|
| 396 |
+
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
|
| 397 |
+
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
|
| 398 |
+
_is_async = kwargs.pop("aget_interaction", False) is True
|
| 399 |
+
|
| 400 |
+
litellm_params = GenericLiteLLMParams(**kwargs)
|
| 401 |
+
|
| 402 |
+
interactions_api_config = get_provider_interactions_api_config(
|
| 403 |
+
provider=custom_llm_provider,
|
| 404 |
+
)
|
| 405 |
+
|
| 406 |
+
if interactions_api_config is None:
|
| 407 |
+
raise ValueError(
|
| 408 |
+
f"Interactions API not supported for: {custom_llm_provider}"
|
| 409 |
+
)
|
| 410 |
+
|
| 411 |
+
litellm_logging_obj.update_from_kwargs(
|
| 412 |
+
kwargs=kwargs,
|
| 413 |
+
model=None,
|
| 414 |
+
optional_params={"interaction_id": interaction_id},
|
| 415 |
+
litellm_params={"litellm_call_id": litellm_call_id},
|
| 416 |
+
custom_llm_provider=custom_llm_provider,
|
| 417 |
+
)
|
| 418 |
+
|
| 419 |
+
return interactions_http_handler.get_interaction(
|
| 420 |
+
interaction_id=interaction_id,
|
| 421 |
+
interactions_api_config=interactions_api_config,
|
| 422 |
+
custom_llm_provider=custom_llm_provider,
|
| 423 |
+
litellm_params=litellm_params,
|
| 424 |
+
logging_obj=litellm_logging_obj,
|
| 425 |
+
extra_headers=extra_headers,
|
| 426 |
+
timeout=timeout,
|
| 427 |
+
_is_async=_is_async,
|
| 428 |
+
)
|
| 429 |
+
except Exception as e:
|
| 430 |
+
raise litellm.exception_type(
|
| 431 |
+
model=None,
|
| 432 |
+
custom_llm_provider=custom_llm_provider,
|
| 433 |
+
original_exception=e,
|
| 434 |
+
completion_kwargs=local_vars,
|
| 435 |
+
extra_kwargs=kwargs,
|
| 436 |
+
)
|
| 437 |
+
|
| 438 |
+
|
| 439 |
+
# ============================================================
|
| 440 |
+
# SDK Methods - DELETE INTERACTION
|
| 441 |
+
# ============================================================
|
| 442 |
+
|
| 443 |
+
|
| 444 |
+
@client
|
| 445 |
+
async def adelete(
|
| 446 |
+
interaction_id: str,
|
| 447 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 448 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 449 |
+
custom_llm_provider: Optional[str] = None,
|
| 450 |
+
**kwargs,
|
| 451 |
+
) -> DeleteInteractionResult:
|
| 452 |
+
"""Async: Delete an interaction by its ID."""
|
| 453 |
+
local_vars = locals()
|
| 454 |
+
try:
|
| 455 |
+
loop = asyncio.get_event_loop()
|
| 456 |
+
kwargs["adelete_interaction"] = True
|
| 457 |
+
|
| 458 |
+
func = partial(
|
| 459 |
+
delete,
|
| 460 |
+
interaction_id=interaction_id,
|
| 461 |
+
extra_headers=extra_headers,
|
| 462 |
+
timeout=timeout,
|
| 463 |
+
custom_llm_provider=custom_llm_provider or "gemini",
|
| 464 |
+
**kwargs,
|
| 465 |
+
)
|
| 466 |
+
|
| 467 |
+
ctx = contextvars.copy_context()
|
| 468 |
+
func_with_context = partial(ctx.run, func)
|
| 469 |
+
init_response = await loop.run_in_executor(None, func_with_context)
|
| 470 |
+
|
| 471 |
+
if asyncio.iscoroutine(init_response):
|
| 472 |
+
response = await init_response
|
| 473 |
+
else:
|
| 474 |
+
response = init_response
|
| 475 |
+
|
| 476 |
+
return response # type: ignore
|
| 477 |
+
except Exception as e:
|
| 478 |
+
raise litellm.exception_type(
|
| 479 |
+
model=None,
|
| 480 |
+
custom_llm_provider=custom_llm_provider or "gemini",
|
| 481 |
+
original_exception=e,
|
| 482 |
+
completion_kwargs=local_vars,
|
| 483 |
+
extra_kwargs=kwargs,
|
| 484 |
+
)
|
| 485 |
+
|
| 486 |
+
|
| 487 |
+
@client
|
| 488 |
+
def delete(
|
| 489 |
+
interaction_id: str,
|
| 490 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 491 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 492 |
+
custom_llm_provider: Optional[str] = None,
|
| 493 |
+
**kwargs,
|
| 494 |
+
) -> Union[DeleteInteractionResult, Coroutine[Any, Any, DeleteInteractionResult]]:
|
| 495 |
+
"""Sync: Delete an interaction by its ID."""
|
| 496 |
+
local_vars = locals()
|
| 497 |
+
custom_llm_provider = custom_llm_provider or "gemini"
|
| 498 |
+
|
| 499 |
+
try:
|
| 500 |
+
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
|
| 501 |
+
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
|
| 502 |
+
_is_async = kwargs.pop("adelete_interaction", False) is True
|
| 503 |
+
|
| 504 |
+
litellm_params = GenericLiteLLMParams(**kwargs)
|
| 505 |
+
|
| 506 |
+
interactions_api_config = get_provider_interactions_api_config(
|
| 507 |
+
provider=custom_llm_provider,
|
| 508 |
+
)
|
| 509 |
+
|
| 510 |
+
if interactions_api_config is None:
|
| 511 |
+
raise ValueError(
|
| 512 |
+
f"Interactions API not supported for: {custom_llm_provider}"
|
| 513 |
+
)
|
| 514 |
+
|
| 515 |
+
litellm_logging_obj.update_from_kwargs(
|
| 516 |
+
kwargs=kwargs,
|
| 517 |
+
model=None,
|
| 518 |
+
optional_params={"interaction_id": interaction_id},
|
| 519 |
+
litellm_params={"litellm_call_id": litellm_call_id},
|
| 520 |
+
custom_llm_provider=custom_llm_provider,
|
| 521 |
+
)
|
| 522 |
+
|
| 523 |
+
return interactions_http_handler.delete_interaction(
|
| 524 |
+
interaction_id=interaction_id,
|
| 525 |
+
interactions_api_config=interactions_api_config,
|
| 526 |
+
custom_llm_provider=custom_llm_provider,
|
| 527 |
+
litellm_params=litellm_params,
|
| 528 |
+
logging_obj=litellm_logging_obj,
|
| 529 |
+
extra_headers=extra_headers,
|
| 530 |
+
timeout=timeout,
|
| 531 |
+
_is_async=_is_async,
|
| 532 |
+
)
|
| 533 |
+
except Exception as e:
|
| 534 |
+
raise litellm.exception_type(
|
| 535 |
+
model=None,
|
| 536 |
+
custom_llm_provider=custom_llm_provider,
|
| 537 |
+
original_exception=e,
|
| 538 |
+
completion_kwargs=local_vars,
|
| 539 |
+
extra_kwargs=kwargs,
|
| 540 |
+
)
|
| 541 |
+
|
| 542 |
+
|
| 543 |
+
# ============================================================
|
| 544 |
+
# SDK Methods - CANCEL INTERACTION
|
| 545 |
+
# ============================================================
|
| 546 |
+
|
| 547 |
+
|
| 548 |
+
@client
|
| 549 |
+
async def acancel(
|
| 550 |
+
interaction_id: str,
|
| 551 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 552 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 553 |
+
custom_llm_provider: Optional[str] = None,
|
| 554 |
+
**kwargs,
|
| 555 |
+
) -> CancelInteractionResult:
|
| 556 |
+
"""Async: Cancel an interaction by its ID."""
|
| 557 |
+
local_vars = locals()
|
| 558 |
+
try:
|
| 559 |
+
loop = asyncio.get_event_loop()
|
| 560 |
+
kwargs["acancel_interaction"] = True
|
| 561 |
+
|
| 562 |
+
func = partial(
|
| 563 |
+
cancel,
|
| 564 |
+
interaction_id=interaction_id,
|
| 565 |
+
extra_headers=extra_headers,
|
| 566 |
+
timeout=timeout,
|
| 567 |
+
custom_llm_provider=custom_llm_provider or "gemini",
|
| 568 |
+
**kwargs,
|
| 569 |
+
)
|
| 570 |
+
|
| 571 |
+
ctx = contextvars.copy_context()
|
| 572 |
+
func_with_context = partial(ctx.run, func)
|
| 573 |
+
init_response = await loop.run_in_executor(None, func_with_context)
|
| 574 |
+
|
| 575 |
+
if asyncio.iscoroutine(init_response):
|
| 576 |
+
response = await init_response
|
| 577 |
+
else:
|
| 578 |
+
response = init_response
|
| 579 |
+
|
| 580 |
+
return response # type: ignore
|
| 581 |
+
except Exception as e:
|
| 582 |
+
raise litellm.exception_type(
|
| 583 |
+
model=None,
|
| 584 |
+
custom_llm_provider=custom_llm_provider or "gemini",
|
| 585 |
+
original_exception=e,
|
| 586 |
+
completion_kwargs=local_vars,
|
| 587 |
+
extra_kwargs=kwargs,
|
| 588 |
+
)
|
| 589 |
+
|
| 590 |
+
|
| 591 |
+
@client
|
| 592 |
+
def cancel(
|
| 593 |
+
interaction_id: str,
|
| 594 |
+
extra_headers: Optional[Dict[str, Any]] = None,
|
| 595 |
+
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
| 596 |
+
custom_llm_provider: Optional[str] = None,
|
| 597 |
+
**kwargs,
|
| 598 |
+
) -> Union[CancelInteractionResult, Coroutine[Any, Any, CancelInteractionResult]]:
|
| 599 |
+
"""Sync: Cancel an interaction by its ID."""
|
| 600 |
+
local_vars = locals()
|
| 601 |
+
custom_llm_provider = custom_llm_provider or "gemini"
|
| 602 |
+
|
| 603 |
+
try:
|
| 604 |
+
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
|
| 605 |
+
litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
|
| 606 |
+
_is_async = kwargs.pop("acancel_interaction", False) is True
|
| 607 |
+
|
| 608 |
+
litellm_params = GenericLiteLLMParams(**kwargs)
|
| 609 |
+
|
| 610 |
+
interactions_api_config = get_provider_interactions_api_config(
|
| 611 |
+
provider=custom_llm_provider,
|
| 612 |
+
)
|
| 613 |
+
|
| 614 |
+
if interactions_api_config is None:
|
| 615 |
+
raise ValueError(
|
| 616 |
+
f"Interactions API not supported for: {custom_llm_provider}"
|
| 617 |
+
)
|
| 618 |
+
|
| 619 |
+
litellm_logging_obj.update_from_kwargs(
|
| 620 |
+
kwargs=kwargs,
|
| 621 |
+
model=None,
|
| 622 |
+
optional_params={"interaction_id": interaction_id},
|
| 623 |
+
litellm_params={"litellm_call_id": litellm_call_id},
|
| 624 |
+
custom_llm_provider=custom_llm_provider,
|
| 625 |
+
)
|
| 626 |
+
|
| 627 |
+
return interactions_http_handler.cancel_interaction(
|
| 628 |
+
interaction_id=interaction_id,
|
| 629 |
+
interactions_api_config=interactions_api_config,
|
| 630 |
+
custom_llm_provider=custom_llm_provider,
|
| 631 |
+
litellm_params=litellm_params,
|
| 632 |
+
logging_obj=litellm_logging_obj,
|
| 633 |
+
extra_headers=extra_headers,
|
| 634 |
+
timeout=timeout,
|
| 635 |
+
_is_async=_is_async,
|
| 636 |
+
)
|
| 637 |
+
except Exception as e:
|
| 638 |
+
raise litellm.exception_type(
|
| 639 |
+
model=None,
|
| 640 |
+
custom_llm_provider=custom_llm_provider,
|
| 641 |
+
original_exception=e,
|
| 642 |
+
completion_kwargs=local_vars,
|
| 643 |
+
extra_kwargs=kwargs,
|
| 644 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/streaming_iterator.py
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Streaming iterators for the Interactions API.
|
| 3 |
+
|
| 4 |
+
This module provides streaming iterators that properly stream SSE responses
|
| 5 |
+
from the Google Interactions API, similar to the responses API streaming iterator.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import asyncio
|
| 9 |
+
import json
|
| 10 |
+
from datetime import datetime
|
| 11 |
+
from typing import Any, Dict, Optional
|
| 12 |
+
|
| 13 |
+
import httpx
|
| 14 |
+
|
| 15 |
+
from litellm._logging import verbose_logger
|
| 16 |
+
from litellm.constants import STREAM_SSE_DONE_STRING
|
| 17 |
+
from litellm.litellm_core_utils.asyncify import run_async_function
|
| 18 |
+
from litellm.litellm_core_utils.core_helpers import process_response_headers
|
| 19 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
| 20 |
+
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
|
| 21 |
+
from litellm.litellm_core_utils.thread_pool_executor import executor
|
| 22 |
+
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
|
| 23 |
+
from litellm.types.interactions import (
|
| 24 |
+
InteractionsAPIStreamingResponse,
|
| 25 |
+
)
|
| 26 |
+
from litellm.utils import CustomStreamWrapper
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class BaseInteractionsAPIStreamingIterator:
|
| 30 |
+
"""
|
| 31 |
+
Base class for streaming iterators that process responses from the Interactions API.
|
| 32 |
+
|
| 33 |
+
This class contains shared logic for both synchronous and asynchronous iterators.
|
| 34 |
+
"""
|
| 35 |
+
|
| 36 |
+
def __init__(
|
| 37 |
+
self,
|
| 38 |
+
response: httpx.Response,
|
| 39 |
+
model: Optional[str],
|
| 40 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 41 |
+
logging_obj: LiteLLMLoggingObj,
|
| 42 |
+
litellm_metadata: Optional[Dict[str, Any]] = None,
|
| 43 |
+
custom_llm_provider: Optional[str] = None,
|
| 44 |
+
):
|
| 45 |
+
self.response = response
|
| 46 |
+
self.model = model
|
| 47 |
+
self.logging_obj = logging_obj
|
| 48 |
+
self.finished = False
|
| 49 |
+
self.interactions_api_config = interactions_api_config
|
| 50 |
+
self.completed_response: Optional[InteractionsAPIStreamingResponse] = None
|
| 51 |
+
self.start_time = datetime.now()
|
| 52 |
+
|
| 53 |
+
# set request kwargs
|
| 54 |
+
self.litellm_metadata = litellm_metadata
|
| 55 |
+
self.custom_llm_provider = custom_llm_provider
|
| 56 |
+
|
| 57 |
+
# set hidden params for response headers
|
| 58 |
+
_api_base = get_api_base(
|
| 59 |
+
model=model or "",
|
| 60 |
+
optional_params=self.logging_obj.model_call_details.get(
|
| 61 |
+
"litellm_params", {}
|
| 62 |
+
),
|
| 63 |
+
)
|
| 64 |
+
_model_info: Dict = (
|
| 65 |
+
litellm_metadata.get("model_info", {}) if litellm_metadata else {}
|
| 66 |
+
)
|
| 67 |
+
self._hidden_params = {
|
| 68 |
+
"model_id": _model_info.get("id", None),
|
| 69 |
+
"api_base": _api_base,
|
| 70 |
+
}
|
| 71 |
+
self._hidden_params["additional_headers"] = process_response_headers(
|
| 72 |
+
self.response.headers or {}
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
def _process_chunk(self, chunk: str) -> Optional[InteractionsAPIStreamingResponse]:
|
| 76 |
+
"""Process a single chunk of data from the stream."""
|
| 77 |
+
if not chunk:
|
| 78 |
+
return None
|
| 79 |
+
|
| 80 |
+
# Handle SSE format (data: {...})
|
| 81 |
+
stripped_chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk)
|
| 82 |
+
if stripped_chunk is None:
|
| 83 |
+
return None
|
| 84 |
+
|
| 85 |
+
# Handle "[DONE]" marker
|
| 86 |
+
if stripped_chunk == STREAM_SSE_DONE_STRING:
|
| 87 |
+
self.finished = True
|
| 88 |
+
return None
|
| 89 |
+
|
| 90 |
+
try:
|
| 91 |
+
# Parse the JSON chunk
|
| 92 |
+
parsed_chunk = json.loads(stripped_chunk)
|
| 93 |
+
|
| 94 |
+
# Format as InteractionsAPIStreamingResponse
|
| 95 |
+
if isinstance(parsed_chunk, dict):
|
| 96 |
+
streaming_response = (
|
| 97 |
+
self.interactions_api_config.transform_streaming_response(
|
| 98 |
+
model=self.model,
|
| 99 |
+
parsed_chunk=parsed_chunk,
|
| 100 |
+
logging_obj=self.logging_obj,
|
| 101 |
+
)
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
# Store the completed response (check for status=completed)
|
| 105 |
+
if (
|
| 106 |
+
streaming_response
|
| 107 |
+
and getattr(streaming_response, "status", None) == "completed"
|
| 108 |
+
):
|
| 109 |
+
self.completed_response = streaming_response
|
| 110 |
+
self._handle_logging_completed_response()
|
| 111 |
+
|
| 112 |
+
return streaming_response
|
| 113 |
+
|
| 114 |
+
return None
|
| 115 |
+
except json.JSONDecodeError:
|
| 116 |
+
# If we can't parse the chunk, continue
|
| 117 |
+
verbose_logger.debug(
|
| 118 |
+
f"Failed to parse streaming chunk: {stripped_chunk[:200]}..."
|
| 119 |
+
)
|
| 120 |
+
return None
|
| 121 |
+
|
| 122 |
+
def _handle_logging_completed_response(self):
|
| 123 |
+
"""Base implementation - should be overridden by subclasses."""
|
| 124 |
+
pass
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
|
| 128 |
+
"""
|
| 129 |
+
Async iterator for processing streaming responses from the Interactions API.
|
| 130 |
+
"""
|
| 131 |
+
|
| 132 |
+
def __init__(
|
| 133 |
+
self,
|
| 134 |
+
response: httpx.Response,
|
| 135 |
+
model: Optional[str],
|
| 136 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 137 |
+
logging_obj: LiteLLMLoggingObj,
|
| 138 |
+
litellm_metadata: Optional[Dict[str, Any]] = None,
|
| 139 |
+
custom_llm_provider: Optional[str] = None,
|
| 140 |
+
):
|
| 141 |
+
super().__init__(
|
| 142 |
+
response=response,
|
| 143 |
+
model=model,
|
| 144 |
+
interactions_api_config=interactions_api_config,
|
| 145 |
+
logging_obj=logging_obj,
|
| 146 |
+
litellm_metadata=litellm_metadata,
|
| 147 |
+
custom_llm_provider=custom_llm_provider,
|
| 148 |
+
)
|
| 149 |
+
self.stream_iterator = response.aiter_lines()
|
| 150 |
+
|
| 151 |
+
def __aiter__(self):
|
| 152 |
+
return self
|
| 153 |
+
|
| 154 |
+
async def __anext__(self) -> InteractionsAPIStreamingResponse:
|
| 155 |
+
try:
|
| 156 |
+
while True:
|
| 157 |
+
# Get the next chunk from the stream
|
| 158 |
+
try:
|
| 159 |
+
chunk = await self.stream_iterator.__anext__()
|
| 160 |
+
except StopAsyncIteration:
|
| 161 |
+
self.finished = True
|
| 162 |
+
raise StopAsyncIteration
|
| 163 |
+
|
| 164 |
+
result = self._process_chunk(chunk)
|
| 165 |
+
|
| 166 |
+
if self.finished:
|
| 167 |
+
raise StopAsyncIteration
|
| 168 |
+
elif result is not None:
|
| 169 |
+
return result
|
| 170 |
+
# If result is None, continue the loop to get the next chunk
|
| 171 |
+
|
| 172 |
+
except httpx.HTTPError as e:
|
| 173 |
+
# Handle HTTP errors
|
| 174 |
+
self.finished = True
|
| 175 |
+
raise e
|
| 176 |
+
|
| 177 |
+
def _handle_logging_completed_response(self):
|
| 178 |
+
"""Handle logging for completed responses in async context."""
|
| 179 |
+
import copy
|
| 180 |
+
|
| 181 |
+
logging_response = copy.deepcopy(self.completed_response)
|
| 182 |
+
|
| 183 |
+
asyncio.create_task(
|
| 184 |
+
self.logging_obj.async_success_handler(
|
| 185 |
+
result=logging_response,
|
| 186 |
+
start_time=self.start_time,
|
| 187 |
+
end_time=datetime.now(),
|
| 188 |
+
cache_hit=None,
|
| 189 |
+
)
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
executor.submit(
|
| 193 |
+
self.logging_obj.success_handler,
|
| 194 |
+
result=logging_response,
|
| 195 |
+
cache_hit=None,
|
| 196 |
+
start_time=self.start_time,
|
| 197 |
+
end_time=datetime.now(),
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
|
| 202 |
+
"""
|
| 203 |
+
Synchronous iterator for processing streaming responses from the Interactions API.
|
| 204 |
+
"""
|
| 205 |
+
|
| 206 |
+
def __init__(
|
| 207 |
+
self,
|
| 208 |
+
response: httpx.Response,
|
| 209 |
+
model: Optional[str],
|
| 210 |
+
interactions_api_config: BaseInteractionsAPIConfig,
|
| 211 |
+
logging_obj: LiteLLMLoggingObj,
|
| 212 |
+
litellm_metadata: Optional[Dict[str, Any]] = None,
|
| 213 |
+
custom_llm_provider: Optional[str] = None,
|
| 214 |
+
):
|
| 215 |
+
super().__init__(
|
| 216 |
+
response=response,
|
| 217 |
+
model=model,
|
| 218 |
+
interactions_api_config=interactions_api_config,
|
| 219 |
+
logging_obj=logging_obj,
|
| 220 |
+
litellm_metadata=litellm_metadata,
|
| 221 |
+
custom_llm_provider=custom_llm_provider,
|
| 222 |
+
)
|
| 223 |
+
self.stream_iterator = response.iter_lines()
|
| 224 |
+
|
| 225 |
+
def __iter__(self):
|
| 226 |
+
return self
|
| 227 |
+
|
| 228 |
+
def __next__(self) -> InteractionsAPIStreamingResponse:
|
| 229 |
+
try:
|
| 230 |
+
while True:
|
| 231 |
+
# Get the next chunk from the stream
|
| 232 |
+
try:
|
| 233 |
+
chunk = next(self.stream_iterator)
|
| 234 |
+
except StopIteration:
|
| 235 |
+
self.finished = True
|
| 236 |
+
raise StopIteration
|
| 237 |
+
|
| 238 |
+
result = self._process_chunk(chunk)
|
| 239 |
+
|
| 240 |
+
if self.finished:
|
| 241 |
+
raise StopIteration
|
| 242 |
+
elif result is not None:
|
| 243 |
+
return result
|
| 244 |
+
# If result is None, continue the loop to get the next chunk
|
| 245 |
+
|
| 246 |
+
except httpx.HTTPError as e:
|
| 247 |
+
# Handle HTTP errors
|
| 248 |
+
self.finished = True
|
| 249 |
+
raise e
|
| 250 |
+
|
| 251 |
+
def _handle_logging_completed_response(self):
|
| 252 |
+
"""Handle logging for completed responses in sync context."""
|
| 253 |
+
import copy
|
| 254 |
+
|
| 255 |
+
logging_response = copy.deepcopy(self.completed_response)
|
| 256 |
+
|
| 257 |
+
run_async_function(
|
| 258 |
+
async_function=self.logging_obj.async_success_handler,
|
| 259 |
+
result=logging_response,
|
| 260 |
+
start_time=self.start_time,
|
| 261 |
+
end_time=datetime.now(),
|
| 262 |
+
cache_hit=None,
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
+
executor.submit(
|
| 266 |
+
self.logging_obj.success_handler,
|
| 267 |
+
result=logging_response,
|
| 268 |
+
cache_hit=None,
|
| 269 |
+
start_time=self.start_time,
|
| 270 |
+
end_time=datetime.now(),
|
| 271 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/utils.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Utility functions for Interactions API.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from typing import Any, Dict, Optional, cast
|
| 6 |
+
|
| 7 |
+
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
|
| 8 |
+
from litellm.types.interactions import InteractionsAPIOptionalRequestParams
|
| 9 |
+
|
| 10 |
+
# Valid optional parameter keys per OpenAPI spec
|
| 11 |
+
INTERACTIONS_API_OPTIONAL_PARAMS = {
|
| 12 |
+
"tools",
|
| 13 |
+
"system_instruction",
|
| 14 |
+
"generation_config",
|
| 15 |
+
"stream",
|
| 16 |
+
"store",
|
| 17 |
+
"background",
|
| 18 |
+
"response_modalities",
|
| 19 |
+
"response_format",
|
| 20 |
+
"response_mime_type",
|
| 21 |
+
"previous_interaction_id",
|
| 22 |
+
"agent_config",
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def get_provider_interactions_api_config(
|
| 27 |
+
provider: str,
|
| 28 |
+
model: Optional[str] = None,
|
| 29 |
+
) -> Optional[BaseInteractionsAPIConfig]:
|
| 30 |
+
"""
|
| 31 |
+
Get the interactions API config for the given provider.
|
| 32 |
+
|
| 33 |
+
Args:
|
| 34 |
+
provider: The LLM provider name
|
| 35 |
+
model: Optional model name
|
| 36 |
+
|
| 37 |
+
Returns:
|
| 38 |
+
The provider-specific interactions API config, or None if not supported
|
| 39 |
+
"""
|
| 40 |
+
from litellm.types.utils import LlmProviders
|
| 41 |
+
|
| 42 |
+
if provider == LlmProviders.GEMINI.value or provider == "gemini":
|
| 43 |
+
from litellm.llms.gemini.interactions.transformation import (
|
| 44 |
+
GoogleAIStudioInteractionsConfig,
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
return GoogleAIStudioInteractionsConfig()
|
| 48 |
+
|
| 49 |
+
return None
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class InteractionsAPIRequestUtils:
|
| 53 |
+
"""Helper utils for constructing Interactions API requests."""
|
| 54 |
+
|
| 55 |
+
@staticmethod
|
| 56 |
+
def get_requested_interactions_api_optional_params(
|
| 57 |
+
params: Dict[str, Any],
|
| 58 |
+
) -> InteractionsAPIOptionalRequestParams:
|
| 59 |
+
"""
|
| 60 |
+
Filter parameters to only include valid optional params per OpenAPI spec.
|
| 61 |
+
|
| 62 |
+
Args:
|
| 63 |
+
params: Dictionary of parameters to filter (typically from locals())
|
| 64 |
+
|
| 65 |
+
Returns:
|
| 66 |
+
Dict with only the valid optional parameters
|
| 67 |
+
"""
|
| 68 |
+
from litellm.utils import PreProcessNonDefaultParams
|
| 69 |
+
|
| 70 |
+
custom_llm_provider = params.pop("custom_llm_provider", None)
|
| 71 |
+
special_params = params.pop("kwargs", {})
|
| 72 |
+
additional_drop_params = params.pop("additional_drop_params", None)
|
| 73 |
+
|
| 74 |
+
non_default_params = (
|
| 75 |
+
PreProcessNonDefaultParams.base_pre_process_non_default_params(
|
| 76 |
+
passed_params=params,
|
| 77 |
+
special_params=special_params,
|
| 78 |
+
custom_llm_provider=custom_llm_provider,
|
| 79 |
+
additional_drop_params=additional_drop_params,
|
| 80 |
+
default_param_values={
|
| 81 |
+
k: None for k in INTERACTIONS_API_OPTIONAL_PARAMS
|
| 82 |
+
},
|
| 83 |
+
additional_endpoint_specific_params=["input", "model", "agent"],
|
| 84 |
+
)
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
return cast(InteractionsAPIOptionalRequestParams, non_default_params)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/README.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## Folder Contents
|
| 2 |
+
|
| 3 |
+
This folder contains general-purpose utilities that are used in multiple places in the codebase.
|
| 4 |
+
|
| 5 |
+
Core files:
|
| 6 |
+
- `streaming_handler.py`: The core streaming logic + streaming related helper utils
|
| 7 |
+
- `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`.
|
| 8 |
+
- `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types.
|
| 9 |
+
- `default_encoding.py`: code for loading the default encoding (tiktoken)
|
| 10 |
+
- `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name.
|
| 11 |
+
- `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
|
| 12 |
+
- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
|
| 13 |
+
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/api_route_to_call_types.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Dictionary mapping API routes to their corresponding CallTypes in LiteLLM.
|
| 3 |
+
|
| 4 |
+
This dictionary maps each API endpoint to the CallTypes that can be used for that route.
|
| 5 |
+
Each route can have both async (prefixed with 'a') and sync call types.
|
| 6 |
+
|
| 7 |
+
Route patterns may contain placeholders like {agent_id}, {model}, {batch_id}; these
|
| 8 |
+
match a single path segment when resolving call types for a concrete path.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from typing import List, Optional
|
| 12 |
+
|
| 13 |
+
from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _route_matches_pattern(route: str, pattern: str) -> bool:
|
| 17 |
+
"""
|
| 18 |
+
Return True if the concrete route matches the pattern.
|
| 19 |
+
Pattern segments like {param} match any single path segment.
|
| 20 |
+
"""
|
| 21 |
+
route_parts = route.strip("/").split("/")
|
| 22 |
+
pattern_parts = pattern.strip("/").split("/")
|
| 23 |
+
if len(route_parts) != len(pattern_parts):
|
| 24 |
+
return False
|
| 25 |
+
for r, p in zip(route_parts, pattern_parts):
|
| 26 |
+
if p.startswith("{") and p.endswith("}"):
|
| 27 |
+
continue
|
| 28 |
+
if r != p:
|
| 29 |
+
return False
|
| 30 |
+
return True
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def get_call_types_for_route(route: str) -> Optional[List[CallTypes]]:
|
| 34 |
+
"""
|
| 35 |
+
Get the list of CallTypes for a given API route.
|
| 36 |
+
|
| 37 |
+
Supports both exact keys and dynamic patterns (e.g. /a2a/my-agent/message/send
|
| 38 |
+
matches /a2a/{agent_id}/message/send).
|
| 39 |
+
|
| 40 |
+
Args:
|
| 41 |
+
route: API route path (e.g., "/chat/completions" or "/a2a/my-pydantic-agent/message/send")
|
| 42 |
+
|
| 43 |
+
Returns:
|
| 44 |
+
List of CallTypes for that route, or None if route not found
|
| 45 |
+
"""
|
| 46 |
+
exact = API_ROUTE_TO_CALL_TYPES.get(route, None)
|
| 47 |
+
if exact is not None:
|
| 48 |
+
return exact
|
| 49 |
+
for pattern, call_types in API_ROUTE_TO_CALL_TYPES.items():
|
| 50 |
+
if _route_matches_pattern(route, pattern):
|
| 51 |
+
return call_types
|
| 52 |
+
return None
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def get_routes_for_call_type(call_type: CallTypes) -> list:
|
| 56 |
+
"""
|
| 57 |
+
Get all routes that use a specific CallType.
|
| 58 |
+
|
| 59 |
+
Args:
|
| 60 |
+
call_type: The CallType to search for
|
| 61 |
+
|
| 62 |
+
Returns:
|
| 63 |
+
List of routes that use this CallType
|
| 64 |
+
"""
|
| 65 |
+
routes = []
|
| 66 |
+
for route, types in API_ROUTE_TO_CALL_TYPES.items():
|
| 67 |
+
if call_type in types:
|
| 68 |
+
routes.append(route)
|
| 69 |
+
return routes
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/app_crypto.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import base64
|
| 2 |
+
import json
|
| 3 |
+
import os
|
| 4 |
+
from typing import Optional
|
| 5 |
+
|
| 6 |
+
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class AppCrypto:
|
| 10 |
+
def __init__(self, master_key: bytes):
|
| 11 |
+
if len(master_key) != 32:
|
| 12 |
+
raise ValueError("Master key must be 32 bytes for AES-256-GCM")
|
| 13 |
+
self.key = master_key
|
| 14 |
+
|
| 15 |
+
def encrypt_json(self, data: dict, aad: Optional[bytes] = None) -> dict:
|
| 16 |
+
aes = AESGCM(self.key)
|
| 17 |
+
nonce = os.urandom(12)
|
| 18 |
+
plaintext = json.dumps(data).encode("utf-8")
|
| 19 |
+
ct = aes.encrypt(nonce, plaintext, aad)
|
| 20 |
+
ciphertext, tag = ct[:-16], ct[-16:]
|
| 21 |
+
return {
|
| 22 |
+
"nonce": base64.b64encode(nonce).decode(),
|
| 23 |
+
"ciphertext": base64.b64encode(ciphertext).decode(),
|
| 24 |
+
"tag": base64.b64encode(tag).decode(),
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
def decrypt_json(self, enc: dict, aad: Optional[bytes] = None) -> dict:
|
| 28 |
+
aes = AESGCM(self.key)
|
| 29 |
+
nonce = base64.b64decode(enc["nonce"])
|
| 30 |
+
ct = base64.b64decode(enc["ciphertext"])
|
| 31 |
+
tag = base64.b64decode(enc["tag"])
|
| 32 |
+
data = aes.decrypt(nonce, ct + tag, aad)
|
| 33 |
+
return json.loads(data.decode())
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/asyncify.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import functools
|
| 3 |
+
from typing import Awaitable, Callable, Optional
|
| 4 |
+
|
| 5 |
+
import anyio
|
| 6 |
+
import anyio.to_thread
|
| 7 |
+
from typing_extensions import ParamSpec, TypeVar
|
| 8 |
+
|
| 9 |
+
T_ParamSpec = ParamSpec("T_ParamSpec")
|
| 10 |
+
T_Retval = TypeVar("T_Retval")
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def function_has_argument(function: Callable, arg_name: str) -> bool:
|
| 14 |
+
"""Helper function to check if a function has a specific argument."""
|
| 15 |
+
import inspect
|
| 16 |
+
|
| 17 |
+
signature = inspect.signature(function)
|
| 18 |
+
return arg_name in signature.parameters
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def asyncify(
|
| 22 |
+
function: Callable[T_ParamSpec, T_Retval],
|
| 23 |
+
*,
|
| 24 |
+
cancellable: bool = False,
|
| 25 |
+
limiter: Optional[anyio.CapacityLimiter] = None,
|
| 26 |
+
) -> Callable[T_ParamSpec, Awaitable[T_Retval]]:
|
| 27 |
+
"""
|
| 28 |
+
Take a blocking function and create an async one that receives the same
|
| 29 |
+
positional and keyword arguments, and that when called, calls the original function
|
| 30 |
+
in a worker thread using `anyio.to_thread.run_sync()`.
|
| 31 |
+
|
| 32 |
+
If the `cancellable` option is enabled and the task waiting for its completion is
|
| 33 |
+
cancelled, the thread will still run its course but its return value (or any raised
|
| 34 |
+
exception) will be ignored.
|
| 35 |
+
|
| 36 |
+
## Arguments
|
| 37 |
+
- `function`: a blocking regular callable (e.g. a function)
|
| 38 |
+
- `cancellable`: `True` to allow cancellation of the operation
|
| 39 |
+
- `limiter`: capacity limiter to use to limit the total amount of threads running
|
| 40 |
+
(if omitted, the default limiter is used)
|
| 41 |
+
|
| 42 |
+
## Return
|
| 43 |
+
An async function that takes the same positional and keyword arguments as the
|
| 44 |
+
original one, that when called runs the same original function in a thread worker
|
| 45 |
+
and returns the result.
|
| 46 |
+
"""
|
| 47 |
+
|
| 48 |
+
async def wrapper(
|
| 49 |
+
*args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs
|
| 50 |
+
) -> T_Retval:
|
| 51 |
+
partial_f = functools.partial(function, *args, **kwargs)
|
| 52 |
+
|
| 53 |
+
# In `v4.1.0` anyio added the `abandon_on_cancel` argument and deprecated the old
|
| 54 |
+
# `cancellable` argument, so we need to use the new `abandon_on_cancel` to avoid
|
| 55 |
+
# surfacing deprecation warnings.
|
| 56 |
+
if function_has_argument(anyio.to_thread.run_sync, "abandon_on_cancel"):
|
| 57 |
+
return await anyio.to_thread.run_sync(
|
| 58 |
+
partial_f,
|
| 59 |
+
abandon_on_cancel=cancellable,
|
| 60 |
+
limiter=limiter,
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
return await anyio.to_thread.run_sync(
|
| 64 |
+
partial_f,
|
| 65 |
+
cancellable=cancellable,
|
| 66 |
+
limiter=limiter,
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
return wrapper
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def run_async_function(async_function, *args, **kwargs):
|
| 73 |
+
"""
|
| 74 |
+
Helper utility to run an async function in a sync context.
|
| 75 |
+
Handles the case where there is an existing event loop running.
|
| 76 |
+
|
| 77 |
+
Args:
|
| 78 |
+
async_function (Callable): The async function to run
|
| 79 |
+
*args: Positional arguments to pass to the async function
|
| 80 |
+
**kwargs: Keyword arguments to pass to the async function
|
| 81 |
+
|
| 82 |
+
Returns:
|
| 83 |
+
The result of the async function execution
|
| 84 |
+
|
| 85 |
+
Example:
|
| 86 |
+
```python
|
| 87 |
+
async def my_async_func(x, y):
|
| 88 |
+
return x + y
|
| 89 |
+
|
| 90 |
+
result = run_async_function(my_async_func, 1, 2)
|
| 91 |
+
```
|
| 92 |
+
"""
|
| 93 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 94 |
+
|
| 95 |
+
def run_in_new_loop():
|
| 96 |
+
"""Run the coroutine in a new event loop within this thread."""
|
| 97 |
+
new_loop = asyncio.new_event_loop()
|
| 98 |
+
try:
|
| 99 |
+
asyncio.set_event_loop(new_loop)
|
| 100 |
+
return new_loop.run_until_complete(async_function(*args, **kwargs))
|
| 101 |
+
finally:
|
| 102 |
+
new_loop.close()
|
| 103 |
+
asyncio.set_event_loop(None)
|
| 104 |
+
|
| 105 |
+
try:
|
| 106 |
+
# First, try to get the current event loop
|
| 107 |
+
_ = asyncio.get_running_loop()
|
| 108 |
+
# If we're already in an event loop, run in a separate thread
|
| 109 |
+
# to avoid nested event loop issues
|
| 110 |
+
with ThreadPoolExecutor(max_workers=1) as executor:
|
| 111 |
+
future = executor.submit(run_in_new_loop)
|
| 112 |
+
return future.result()
|
| 113 |
+
|
| 114 |
+
except RuntimeError:
|
| 115 |
+
# No running event loop, we can safely run in this thread
|
| 116 |
+
return run_in_new_loop()
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/audio_utils/audio_health_check.wav
ADDED
|
Binary file (29.2 kB). View file
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/audio_utils/utils.py
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Utils used for litellm.transcription() and litellm.atranscription()
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import hashlib
|
| 6 |
+
import os
|
| 7 |
+
from dataclasses import dataclass
|
| 8 |
+
from typing import Optional
|
| 9 |
+
|
| 10 |
+
from litellm.types.files import get_file_mime_type_from_extension
|
| 11 |
+
from litellm.types.utils import FileTypes
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@dataclass
|
| 15 |
+
class ProcessedAudioFile:
|
| 16 |
+
"""
|
| 17 |
+
Processed audio file data.
|
| 18 |
+
|
| 19 |
+
Attributes:
|
| 20 |
+
file_content: The binary content of the audio file
|
| 21 |
+
filename: The filename (extracted or generated)
|
| 22 |
+
content_type: The MIME type of the audio file
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
file_content: bytes
|
| 26 |
+
filename: str
|
| 27 |
+
content_type: str
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def process_audio_file(audio_file: FileTypes) -> ProcessedAudioFile:
|
| 31 |
+
"""
|
| 32 |
+
Common utility function to process audio files for audio transcription APIs.
|
| 33 |
+
|
| 34 |
+
Handles various input types:
|
| 35 |
+
- File paths (str, os.PathLike)
|
| 36 |
+
- Raw bytes/bytearray
|
| 37 |
+
- Tuples (filename, content, optional content_type)
|
| 38 |
+
- File-like objects with read() method
|
| 39 |
+
|
| 40 |
+
Args:
|
| 41 |
+
audio_file: The audio file input in various formats
|
| 42 |
+
|
| 43 |
+
Returns:
|
| 44 |
+
ProcessedAudioFile: Structured data with file content, filename, and content type
|
| 45 |
+
|
| 46 |
+
Raises:
|
| 47 |
+
ValueError: If audio_file type is unsupported or content cannot be extracted
|
| 48 |
+
"""
|
| 49 |
+
file_content = None
|
| 50 |
+
filename = None
|
| 51 |
+
|
| 52 |
+
if isinstance(audio_file, (bytes, bytearray)):
|
| 53 |
+
# Raw bytes
|
| 54 |
+
filename = "audio.wav"
|
| 55 |
+
file_content = bytes(audio_file)
|
| 56 |
+
elif isinstance(audio_file, (str, os.PathLike)):
|
| 57 |
+
# File path or PathLike
|
| 58 |
+
file_path = str(audio_file)
|
| 59 |
+
with open(file_path, "rb") as f:
|
| 60 |
+
file_content = f.read()
|
| 61 |
+
filename = file_path.split("/")[-1]
|
| 62 |
+
elif isinstance(audio_file, tuple):
|
| 63 |
+
# Tuple format: (filename, content, content_type) or (filename, content)
|
| 64 |
+
if len(audio_file) >= 2:
|
| 65 |
+
filename = audio_file[0] or "audio.wav"
|
| 66 |
+
content = audio_file[1]
|
| 67 |
+
if isinstance(content, (bytes, bytearray)):
|
| 68 |
+
file_content = bytes(content)
|
| 69 |
+
elif isinstance(content, (str, os.PathLike)):
|
| 70 |
+
# File path or PathLike
|
| 71 |
+
with open(str(content), "rb") as f:
|
| 72 |
+
file_content = f.read()
|
| 73 |
+
elif hasattr(content, "read"):
|
| 74 |
+
# File-like object
|
| 75 |
+
file_content = content.read()
|
| 76 |
+
if hasattr(content, "seek"):
|
| 77 |
+
content.seek(0)
|
| 78 |
+
else:
|
| 79 |
+
raise ValueError(f"Unsupported content type in tuple: {type(content)}")
|
| 80 |
+
else:
|
| 81 |
+
raise ValueError("Tuple must have at least 2 elements: (filename, content)")
|
| 82 |
+
elif hasattr(audio_file, "read") and not isinstance(
|
| 83 |
+
audio_file, (str, bytes, bytearray, tuple, os.PathLike)
|
| 84 |
+
):
|
| 85 |
+
# File-like object (IO) - check this after all other types
|
| 86 |
+
filename = getattr(audio_file, "name", "audio.wav")
|
| 87 |
+
file_content = audio_file.read() # type: ignore
|
| 88 |
+
# Reset file pointer if possible
|
| 89 |
+
if hasattr(audio_file, "seek"):
|
| 90 |
+
audio_file.seek(0) # type: ignore
|
| 91 |
+
else:
|
| 92 |
+
raise ValueError(f"Unsupported audio_file type: {type(audio_file)}")
|
| 93 |
+
|
| 94 |
+
if file_content is None:
|
| 95 |
+
raise ValueError("Could not extract file content from audio_file")
|
| 96 |
+
|
| 97 |
+
# Determine content type using LiteLLM's file type utilities
|
| 98 |
+
content_type = "audio/wav" # Default fallback
|
| 99 |
+
if filename:
|
| 100 |
+
try:
|
| 101 |
+
# Extract extension from filename
|
| 102 |
+
extension = filename.split(".")[-1].lower() if "." in filename else "wav"
|
| 103 |
+
content_type = get_file_mime_type_from_extension(extension)
|
| 104 |
+
except ValueError:
|
| 105 |
+
# If extension is not recognized, fallback to audio/wav
|
| 106 |
+
content_type = "audio/wav"
|
| 107 |
+
|
| 108 |
+
return ProcessedAudioFile(
|
| 109 |
+
file_content=file_content, filename=filename, content_type=content_type
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def get_audio_file_name(file_obj: FileTypes) -> str:
|
| 114 |
+
"""
|
| 115 |
+
Safely get the name of a file-like object or return its string representation.
|
| 116 |
+
|
| 117 |
+
Args:
|
| 118 |
+
file_obj (Any): A file-like object or any other object.
|
| 119 |
+
|
| 120 |
+
Returns:
|
| 121 |
+
str: The name of the file if available, otherwise a string representation of the object.
|
| 122 |
+
"""
|
| 123 |
+
if hasattr(file_obj, "name"):
|
| 124 |
+
return getattr(file_obj, "name")
|
| 125 |
+
elif hasattr(file_obj, "__str__"):
|
| 126 |
+
return str(file_obj)
|
| 127 |
+
else:
|
| 128 |
+
return repr(file_obj)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def get_audio_file_content_hash(file_obj: FileTypes) -> str:
|
| 132 |
+
"""
|
| 133 |
+
Compute SHA-256 hash of audio file content for cache keys.
|
| 134 |
+
Falls back to filename hash if content extraction fails.
|
| 135 |
+
"""
|
| 136 |
+
file_content: Optional[bytes] = None
|
| 137 |
+
fallback_filename: Optional[str] = None
|
| 138 |
+
|
| 139 |
+
if isinstance(file_obj, tuple):
|
| 140 |
+
if len(file_obj) < 2:
|
| 141 |
+
fallback_filename = str(file_obj[0]) if len(file_obj) > 0 else None
|
| 142 |
+
else:
|
| 143 |
+
fallback_filename = str(file_obj[0]) if file_obj[0] is not None else None
|
| 144 |
+
file_content_obj = file_obj[1]
|
| 145 |
+
else:
|
| 146 |
+
file_content_obj = file_obj
|
| 147 |
+
fallback_filename = get_audio_file_name(file_obj)
|
| 148 |
+
|
| 149 |
+
try:
|
| 150 |
+
if isinstance(file_content_obj, (bytes, bytearray)):
|
| 151 |
+
file_content = bytes(file_content_obj)
|
| 152 |
+
elif isinstance(file_content_obj, (str, os.PathLike)):
|
| 153 |
+
try:
|
| 154 |
+
with open(str(file_content_obj), "rb") as f:
|
| 155 |
+
file_content = f.read()
|
| 156 |
+
if fallback_filename is None:
|
| 157 |
+
fallback_filename = str(file_content_obj)
|
| 158 |
+
except (OSError, IOError):
|
| 159 |
+
fallback_filename = str(file_content_obj)
|
| 160 |
+
file_content = None
|
| 161 |
+
elif hasattr(file_content_obj, "read"):
|
| 162 |
+
try:
|
| 163 |
+
current_position = (
|
| 164 |
+
file_content_obj.tell()
|
| 165 |
+
if hasattr(file_content_obj, "tell")
|
| 166 |
+
else None
|
| 167 |
+
)
|
| 168 |
+
if hasattr(file_content_obj, "seek"):
|
| 169 |
+
file_content_obj.seek(0)
|
| 170 |
+
file_content = file_content_obj.read() # type: ignore
|
| 171 |
+
if current_position is not None and hasattr(file_content_obj, "seek"):
|
| 172 |
+
file_content_obj.seek(current_position) # type: ignore
|
| 173 |
+
except (OSError, IOError, AttributeError):
|
| 174 |
+
file_content = None
|
| 175 |
+
else:
|
| 176 |
+
file_content = None
|
| 177 |
+
except Exception:
|
| 178 |
+
file_content = None
|
| 179 |
+
|
| 180 |
+
if file_content is not None and isinstance(file_content, bytes):
|
| 181 |
+
try:
|
| 182 |
+
hash_object = hashlib.sha256(file_content)
|
| 183 |
+
return hash_object.hexdigest()
|
| 184 |
+
except Exception:
|
| 185 |
+
pass
|
| 186 |
+
|
| 187 |
+
if fallback_filename:
|
| 188 |
+
hash_object = hashlib.sha256(fallback_filename.encode("utf-8"))
|
| 189 |
+
return hash_object.hexdigest()
|
| 190 |
+
|
| 191 |
+
file_obj_str = str(file_obj)
|
| 192 |
+
hash_object = hashlib.sha256(file_obj_str.encode("utf-8"))
|
| 193 |
+
return hash_object.hexdigest()
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def get_audio_file_for_health_check() -> FileTypes:
|
| 197 |
+
"""
|
| 198 |
+
Get an audio file for health check
|
| 199 |
+
|
| 200 |
+
Returns the content of `audio_health_check.wav` in the same directory as this file
|
| 201 |
+
"""
|
| 202 |
+
pwd = os.path.dirname(os.path.realpath(__file__))
|
| 203 |
+
file_path = os.path.join(pwd, "audio_health_check.wav")
|
| 204 |
+
return open(file_path, "rb")
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def calculate_request_duration(file: FileTypes) -> Optional[float]:
|
| 208 |
+
"""
|
| 209 |
+
Calculate audio duration from file content.
|
| 210 |
+
|
| 211 |
+
Args:
|
| 212 |
+
file: The audio file (can be file path, bytes, or file-like object)
|
| 213 |
+
|
| 214 |
+
Returns:
|
| 215 |
+
Duration in seconds, or None if extraction fails or soundfile is not available
|
| 216 |
+
"""
|
| 217 |
+
try:
|
| 218 |
+
import soundfile as sf
|
| 219 |
+
except ImportError:
|
| 220 |
+
# soundfile not available, cannot extract duration
|
| 221 |
+
return None
|
| 222 |
+
|
| 223 |
+
try:
|
| 224 |
+
import io
|
| 225 |
+
|
| 226 |
+
# Handle different file input types
|
| 227 |
+
file_content: Optional[bytes] = None
|
| 228 |
+
|
| 229 |
+
if isinstance(file, (bytes, bytearray)):
|
| 230 |
+
# Raw bytes
|
| 231 |
+
file_content = bytes(file)
|
| 232 |
+
elif isinstance(file, (str, os.PathLike)):
|
| 233 |
+
# File path
|
| 234 |
+
with open(str(file), "rb") as f:
|
| 235 |
+
file_content = f.read()
|
| 236 |
+
elif isinstance(file, tuple):
|
| 237 |
+
# Tuple format: (filename, content, optional content_type)
|
| 238 |
+
if len(file) >= 2:
|
| 239 |
+
content = file[1]
|
| 240 |
+
if isinstance(content, bytes):
|
| 241 |
+
file_content = content
|
| 242 |
+
elif hasattr(content, "read") and not isinstance(
|
| 243 |
+
content, (str, os.PathLike)
|
| 244 |
+
):
|
| 245 |
+
# File-like object in tuple
|
| 246 |
+
current_pos = getattr(content, "tell", lambda: None)()
|
| 247 |
+
# Seek to start to ensure we read the entire content
|
| 248 |
+
if hasattr(content, "seek"):
|
| 249 |
+
content.seek(0)
|
| 250 |
+
file_content = content.read()
|
| 251 |
+
if current_pos is not None and hasattr(content, "seek"):
|
| 252 |
+
content.seek(current_pos)
|
| 253 |
+
elif hasattr(file, "read") and not isinstance(file, tuple):
|
| 254 |
+
# File-like object (including BytesIO)
|
| 255 |
+
current_position = file.tell() if hasattr(file, "tell") else None
|
| 256 |
+
# Seek to start to ensure we read the entire content
|
| 257 |
+
if hasattr(file, "seek"):
|
| 258 |
+
file.seek(0)
|
| 259 |
+
file_content = file.read()
|
| 260 |
+
# Reset file position if possible
|
| 261 |
+
if current_position is not None and hasattr(file, "seek"):
|
| 262 |
+
file.seek(current_position)
|
| 263 |
+
|
| 264 |
+
if file_content is None or not isinstance(file_content, bytes):
|
| 265 |
+
return None
|
| 266 |
+
|
| 267 |
+
# Extract duration using soundfile
|
| 268 |
+
file_object = io.BytesIO(file_content)
|
| 269 |
+
with sf.SoundFile(file_object) as audio:
|
| 270 |
+
duration = len(audio) / audio.samplerate
|
| 271 |
+
return duration
|
| 272 |
+
|
| 273 |
+
except Exception:
|
| 274 |
+
# Silently fail if duration extraction fails
|
| 275 |
+
return None
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/cached_imports.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Cached imports module for LiteLLM.
|
| 3 |
+
|
| 4 |
+
This module provides cached import functionality to avoid repeated imports
|
| 5 |
+
inside functions that are critical to performance.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import TYPE_CHECKING, Callable, Optional, Type
|
| 9 |
+
|
| 10 |
+
# Type annotations for cached imports
|
| 11 |
+
if TYPE_CHECKING:
|
| 12 |
+
from litellm.litellm_core_utils.litellm_logging import Logging
|
| 13 |
+
from litellm.litellm_core_utils.coroutine_checker import CoroutineChecker
|
| 14 |
+
|
| 15 |
+
# Global cache variables
|
| 16 |
+
_LiteLLMLogging: Optional[Type["Logging"]] = None
|
| 17 |
+
_coroutine_checker: Optional["CoroutineChecker"] = None
|
| 18 |
+
_set_callbacks: Optional[Callable] = None
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def get_litellm_logging_class() -> Type["Logging"]:
|
| 22 |
+
"""Get the cached LiteLLM Logging class, initializing if needed."""
|
| 23 |
+
global _LiteLLMLogging
|
| 24 |
+
if _LiteLLMLogging is not None:
|
| 25 |
+
return _LiteLLMLogging
|
| 26 |
+
from litellm.litellm_core_utils.litellm_logging import Logging
|
| 27 |
+
|
| 28 |
+
_LiteLLMLogging = Logging
|
| 29 |
+
return _LiteLLMLogging
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def get_coroutine_checker() -> "CoroutineChecker":
|
| 33 |
+
"""Get the cached coroutine checker instance, initializing if needed."""
|
| 34 |
+
global _coroutine_checker
|
| 35 |
+
if _coroutine_checker is not None:
|
| 36 |
+
return _coroutine_checker
|
| 37 |
+
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
|
| 38 |
+
|
| 39 |
+
_coroutine_checker = coroutine_checker
|
| 40 |
+
return _coroutine_checker
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def get_set_callbacks() -> Callable:
|
| 44 |
+
"""Get the cached set_callbacks function, initializing if needed."""
|
| 45 |
+
global _set_callbacks
|
| 46 |
+
if _set_callbacks is not None:
|
| 47 |
+
return _set_callbacks
|
| 48 |
+
from litellm.litellm_core_utils.litellm_logging import set_callbacks
|
| 49 |
+
|
| 50 |
+
_set_callbacks = set_callbacks
|
| 51 |
+
return _set_callbacks
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def clear_cached_imports() -> None:
|
| 55 |
+
"""Clear all cached imports. Useful for testing or memory management."""
|
| 56 |
+
global _LiteLLMLogging, _coroutine_checker, _set_callbacks
|
| 57 |
+
_LiteLLMLogging = None
|
| 58 |
+
_coroutine_checker = None
|
| 59 |
+
_set_callbacks = None
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/cli_token_utils.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
CLI Token Utilities
|
| 3 |
+
|
| 4 |
+
SDK-level utilities for reading CLI authentication tokens.
|
| 5 |
+
This module has no dependencies on proxy code and can be safely imported at the SDK level.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
from pathlib import Path
|
| 11 |
+
from typing import Optional
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
def get_cli_token_file_path() -> str:
|
| 15 |
+
"""Get the path to the CLI token file"""
|
| 16 |
+
home_dir = Path.home()
|
| 17 |
+
config_dir = home_dir / ".litellm"
|
| 18 |
+
return str(config_dir / "token.json")
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def load_cli_token() -> Optional[dict]:
|
| 22 |
+
"""Load CLI token data from file"""
|
| 23 |
+
token_file = get_cli_token_file_path()
|
| 24 |
+
if not os.path.exists(token_file):
|
| 25 |
+
return None
|
| 26 |
+
|
| 27 |
+
try:
|
| 28 |
+
with open(token_file, "r") as f:
|
| 29 |
+
return json.load(f)
|
| 30 |
+
except (json.JSONDecodeError, IOError):
|
| 31 |
+
return None
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def get_litellm_gateway_api_key() -> Optional[str]:
|
| 35 |
+
"""
|
| 36 |
+
Get the stored CLI API key for use with LiteLLM SDK.
|
| 37 |
+
|
| 38 |
+
This function reads the token file created by `litellm-proxy login`
|
| 39 |
+
and returns the API key for use in Python scripts.
|
| 40 |
+
|
| 41 |
+
Returns:
|
| 42 |
+
str: The API key if found, None otherwise
|
| 43 |
+
|
| 44 |
+
Example:
|
| 45 |
+
>>> import litellm
|
| 46 |
+
>>> api_key = litellm.get_litellm_gateway_api_key()
|
| 47 |
+
>>> if api_key:
|
| 48 |
+
>>> response = litellm.completion(
|
| 49 |
+
>>> model="gpt-3.5-turbo",
|
| 50 |
+
>>> messages=[{"role": "user", "content": "Hello"}],
|
| 51 |
+
>>> api_key=api_key,
|
| 52 |
+
>>> base_url="https://your-proxy.com/v1"
|
| 53 |
+
>>> )
|
| 54 |
+
"""
|
| 55 |
+
token_data = load_cli_token()
|
| 56 |
+
if token_data and "key" in token_data:
|
| 57 |
+
return token_data["key"]
|
| 58 |
+
return None
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/core_helpers.py
ADDED
|
@@ -0,0 +1,437 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# What is this?
|
| 2 |
+
## Helper utilities
|
| 3 |
+
from typing import TYPE_CHECKING, Any, Iterable, List, Literal, Optional, Union
|
| 4 |
+
|
| 5 |
+
import httpx
|
| 6 |
+
|
| 7 |
+
from litellm._logging import verbose_logger
|
| 8 |
+
from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
|
| 9 |
+
|
| 10 |
+
if TYPE_CHECKING:
|
| 11 |
+
from opentelemetry.trace import Span as _Span
|
| 12 |
+
|
| 13 |
+
from litellm.types.utils import ModelResponseStream
|
| 14 |
+
|
| 15 |
+
Span = Union[_Span, Any]
|
| 16 |
+
else:
|
| 17 |
+
Span = Any
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def safe_divide_seconds(
|
| 21 |
+
seconds: float, denominator: float, default: Optional[float] = None
|
| 22 |
+
) -> Optional[float]:
|
| 23 |
+
"""
|
| 24 |
+
Safely divide seconds by denominator, handling zero division.
|
| 25 |
+
|
| 26 |
+
Args:
|
| 27 |
+
seconds: Time duration in seconds
|
| 28 |
+
denominator: The divisor (e.g., number of tokens)
|
| 29 |
+
default: Value to return if division by zero (defaults to None)
|
| 30 |
+
|
| 31 |
+
Returns:
|
| 32 |
+
The result of the division as a float (seconds per unit), or default if denominator is zero
|
| 33 |
+
"""
|
| 34 |
+
if denominator <= 0:
|
| 35 |
+
return default
|
| 36 |
+
|
| 37 |
+
return float(seconds / denominator)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def safe_divide(
|
| 41 |
+
numerator: Union[int, float],
|
| 42 |
+
denominator: Union[int, float],
|
| 43 |
+
default: Union[int, float] = 0,
|
| 44 |
+
) -> Union[int, float]:
|
| 45 |
+
"""
|
| 46 |
+
Safely divide two numbers, returning a default value if denominator is zero.
|
| 47 |
+
|
| 48 |
+
Args:
|
| 49 |
+
numerator: The number to divide
|
| 50 |
+
denominator: The number to divide by
|
| 51 |
+
default: Value to return if denominator is zero (defaults to 0)
|
| 52 |
+
|
| 53 |
+
Returns:
|
| 54 |
+
The result of numerator/denominator, or default if denominator is zero
|
| 55 |
+
"""
|
| 56 |
+
if denominator == 0:
|
| 57 |
+
return default
|
| 58 |
+
return numerator / denominator
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
_FINISH_REASON_MAP: dict[str, OpenAIChatCompletionFinishReason] = {
|
| 62 |
+
# Anthropic
|
| 63 |
+
"stop_sequence": "stop",
|
| 64 |
+
"end_turn": "stop",
|
| 65 |
+
"max_tokens": "length",
|
| 66 |
+
"tool_use": "tool_calls",
|
| 67 |
+
"refusal": "content_filter",
|
| 68 |
+
"compaction": "length",
|
| 69 |
+
# Cohere
|
| 70 |
+
"COMPLETE": "stop",
|
| 71 |
+
"ERROR_TOXIC": "content_filter",
|
| 72 |
+
"ERROR": "stop",
|
| 73 |
+
# HuggingFace / Together AI
|
| 74 |
+
"eos_token": "stop",
|
| 75 |
+
"eos": "stop",
|
| 76 |
+
# Gemini / Vertex AI
|
| 77 |
+
"STOP": "stop",
|
| 78 |
+
"MAX_TOKENS": "length",
|
| 79 |
+
"SAFETY": "content_filter",
|
| 80 |
+
"RECITATION": "content_filter",
|
| 81 |
+
"FINISH_REASON_UNSPECIFIED": "stop",
|
| 82 |
+
"MALFORMED_FUNCTION_CALL": "stop",
|
| 83 |
+
"LANGUAGE": "content_filter",
|
| 84 |
+
"OTHER": "content_filter",
|
| 85 |
+
"BLOCKLIST": "content_filter",
|
| 86 |
+
"PROHIBITED_CONTENT": "content_filter",
|
| 87 |
+
"SPII": "content_filter",
|
| 88 |
+
"IMAGE_SAFETY": "content_filter",
|
| 89 |
+
"IMAGE_PROHIBITED_CONTENT": "content_filter",
|
| 90 |
+
"TOO_MANY_TOOL_CALLS": "stop",
|
| 91 |
+
"MALFORMED_RESPONSE": "stop",
|
| 92 |
+
# Bedrock
|
| 93 |
+
"guardrail_intervened": "content_filter",
|
| 94 |
+
# OpenAI passthrough
|
| 95 |
+
"stop": "stop",
|
| 96 |
+
"length": "length",
|
| 97 |
+
"tool_calls": "tool_calls",
|
| 98 |
+
"function_call": "function_call",
|
| 99 |
+
"content_filter": "content_filter",
|
| 100 |
+
# Anthropic Sonnet 4
|
| 101 |
+
"content_filtered": "content_filter",
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def map_finish_reason(finish_reason: str) -> OpenAIChatCompletionFinishReason:
|
| 106 |
+
mapped = _FINISH_REASON_MAP.get(finish_reason)
|
| 107 |
+
if mapped is None:
|
| 108 |
+
verbose_logger.warning(
|
| 109 |
+
"Unmapped finish_reason '%s', defaulting to 'stop'", finish_reason
|
| 110 |
+
)
|
| 111 |
+
return "stop"
|
| 112 |
+
return mapped
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def remove_index_from_tool_calls(
|
| 116 |
+
messages: Optional[List[AllMessageValues]],
|
| 117 |
+
):
|
| 118 |
+
if messages is not None:
|
| 119 |
+
for message in messages:
|
| 120 |
+
_tool_calls = message.get("tool_calls")
|
| 121 |
+
if _tool_calls is not None and isinstance(_tool_calls, list):
|
| 122 |
+
for tool_call in _tool_calls:
|
| 123 |
+
if (
|
| 124 |
+
isinstance(tool_call, dict) and "index" in tool_call
|
| 125 |
+
): # Type guard to ensure it's a dict
|
| 126 |
+
tool_call.pop("index", None)
|
| 127 |
+
|
| 128 |
+
return
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def remove_items_at_indices(items: Optional[List[Any]], indices: Iterable[int]) -> None:
|
| 132 |
+
"""Remove items from a list in-place by index"""
|
| 133 |
+
if items is None:
|
| 134 |
+
return
|
| 135 |
+
for index in sorted(set(indices), reverse=True):
|
| 136 |
+
if 0 <= index < len(items):
|
| 137 |
+
items.pop(index)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def add_missing_spend_metadata_to_litellm_metadata(
|
| 141 |
+
litellm_metadata: dict, metadata: dict
|
| 142 |
+
) -> dict:
|
| 143 |
+
"""
|
| 144 |
+
Helper to get litellm metadata for spend tracking
|
| 145 |
+
|
| 146 |
+
PATCH for issue where both `litellm_metadata` and `metadata` are present in the kwargs
|
| 147 |
+
and user_api_key values are in 'metadata'.
|
| 148 |
+
"""
|
| 149 |
+
potential_spend_tracking_metadata_substring = "user_api_key"
|
| 150 |
+
for key, value in metadata.items():
|
| 151 |
+
if potential_spend_tracking_metadata_substring in key:
|
| 152 |
+
litellm_metadata[key] = value
|
| 153 |
+
return litellm_metadata
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def get_metadata_variable_name_from_kwargs(
|
| 157 |
+
kwargs: dict,
|
| 158 |
+
) -> Literal["metadata", "litellm_metadata"]:
|
| 159 |
+
"""
|
| 160 |
+
Helper to return what the "metadata" field should be called in the request data
|
| 161 |
+
|
| 162 |
+
- New endpoints return `litellm_metadata`
|
| 163 |
+
- Old endpoints return `metadata`
|
| 164 |
+
|
| 165 |
+
Context:
|
| 166 |
+
- LiteLLM used `metadata` as an internal field for storing metadata
|
| 167 |
+
- OpenAI then started using this field for their metadata
|
| 168 |
+
- LiteLLM is now moving to using `litellm_metadata` for our metadata
|
| 169 |
+
"""
|
| 170 |
+
return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def get_litellm_metadata_from_kwargs(kwargs: dict):
|
| 174 |
+
"""
|
| 175 |
+
Helper to get litellm metadata from all litellm request kwargs
|
| 176 |
+
|
| 177 |
+
Return `litellm_metadata` if it exists, otherwise return `metadata`
|
| 178 |
+
"""
|
| 179 |
+
litellm_params = kwargs.get("litellm_params", {})
|
| 180 |
+
if litellm_params:
|
| 181 |
+
metadata = litellm_params.get("metadata", {})
|
| 182 |
+
litellm_metadata = litellm_params.get("litellm_metadata", {})
|
| 183 |
+
if litellm_metadata and metadata:
|
| 184 |
+
litellm_metadata = add_missing_spend_metadata_to_litellm_metadata(
|
| 185 |
+
litellm_metadata, metadata
|
| 186 |
+
)
|
| 187 |
+
if litellm_metadata:
|
| 188 |
+
return litellm_metadata
|
| 189 |
+
elif metadata:
|
| 190 |
+
return metadata
|
| 191 |
+
|
| 192 |
+
return {}
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def reconstruct_model_name(
|
| 196 |
+
model_name: str,
|
| 197 |
+
custom_llm_provider: Optional[str],
|
| 198 |
+
metadata: dict,
|
| 199 |
+
) -> str:
|
| 200 |
+
"""Reconstruct full model name with provider prefix for logging."""
|
| 201 |
+
# Check if deployment model name from router metadata is available (has original prefix)
|
| 202 |
+
deployment_model_name = metadata.get("deployment")
|
| 203 |
+
if deployment_model_name and "/" in deployment_model_name:
|
| 204 |
+
# Use the deployment model name which preserves the original provider prefix
|
| 205 |
+
return deployment_model_name
|
| 206 |
+
elif custom_llm_provider and model_name and "/" not in model_name:
|
| 207 |
+
# Only add prefix for Bedrock (not for direct Anthropic API)
|
| 208 |
+
# This ensures Bedrock models get the prefix while direct Anthropic models don't
|
| 209 |
+
if custom_llm_provider == "bedrock":
|
| 210 |
+
return f"{custom_llm_provider}/{model_name}"
|
| 211 |
+
return model_name
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
# Helper functions used for OTEL logging
|
| 215 |
+
def _get_parent_otel_span_from_kwargs(
|
| 216 |
+
kwargs: Optional[dict] = None,
|
| 217 |
+
) -> Union[Span, None]:
|
| 218 |
+
try:
|
| 219 |
+
if kwargs is None:
|
| 220 |
+
return None
|
| 221 |
+
litellm_params = kwargs.get("litellm_params")
|
| 222 |
+
_metadata = kwargs.get("metadata") or {}
|
| 223 |
+
if "litellm_parent_otel_span" in _metadata:
|
| 224 |
+
return _metadata["litellm_parent_otel_span"]
|
| 225 |
+
elif (
|
| 226 |
+
litellm_params is not None
|
| 227 |
+
and litellm_params.get("metadata") is not None
|
| 228 |
+
and "litellm_parent_otel_span" in litellm_params.get("metadata", {})
|
| 229 |
+
):
|
| 230 |
+
return litellm_params["metadata"]["litellm_parent_otel_span"]
|
| 231 |
+
elif "litellm_parent_otel_span" in kwargs:
|
| 232 |
+
return kwargs["litellm_parent_otel_span"]
|
| 233 |
+
return None
|
| 234 |
+
except Exception as e:
|
| 235 |
+
verbose_logger.exception(
|
| 236 |
+
"Error in _get_parent_otel_span_from_kwargs: " + str(e)
|
| 237 |
+
)
|
| 238 |
+
return None
|
| 239 |
+
|
| 240 |
+
|
| 241 |
+
def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> dict:
|
| 242 |
+
from litellm.types.utils import OPENAI_RESPONSE_HEADERS
|
| 243 |
+
|
| 244 |
+
openai_headers = {}
|
| 245 |
+
processed_headers = {}
|
| 246 |
+
additional_headers = {}
|
| 247 |
+
|
| 248 |
+
for k, v in response_headers.items():
|
| 249 |
+
if k in OPENAI_RESPONSE_HEADERS: # return openai-compatible headers
|
| 250 |
+
openai_headers[k] = v
|
| 251 |
+
if k.startswith(
|
| 252 |
+
"llm_provider-"
|
| 253 |
+
): # return raw provider headers (incl. openai-compatible ones)
|
| 254 |
+
processed_headers[k] = v
|
| 255 |
+
else:
|
| 256 |
+
additional_headers["{}-{}".format("llm_provider", k)] = v
|
| 257 |
+
|
| 258 |
+
additional_headers = {
|
| 259 |
+
**openai_headers,
|
| 260 |
+
**processed_headers,
|
| 261 |
+
**additional_headers,
|
| 262 |
+
}
|
| 263 |
+
return additional_headers
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def preserve_upstream_non_openai_attributes(
|
| 267 |
+
model_response: "ModelResponseStream", original_chunk: "ModelResponseStream"
|
| 268 |
+
):
|
| 269 |
+
"""
|
| 270 |
+
Preserve non-OpenAI attributes from the original chunk.
|
| 271 |
+
"""
|
| 272 |
+
# Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
|
| 273 |
+
expected_keys = set(type(model_response).model_fields.keys()).union({"usage"})
|
| 274 |
+
for key, value in original_chunk.model_dump().items():
|
| 275 |
+
if key not in expected_keys:
|
| 276 |
+
setattr(model_response, key, value)
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def safe_deep_copy(data):
|
| 280 |
+
"""
|
| 281 |
+
Safe Deep Copy
|
| 282 |
+
|
| 283 |
+
The LiteLLM request may contain objects that cannot be pickled/deep-copied
|
| 284 |
+
(e.g., tracing spans, locks, clients).
|
| 285 |
+
|
| 286 |
+
This helper deep-copies each top-level key independently; on failure keeps
|
| 287 |
+
original ref
|
| 288 |
+
"""
|
| 289 |
+
import copy
|
| 290 |
+
|
| 291 |
+
import litellm
|
| 292 |
+
|
| 293 |
+
if litellm.safe_memory_mode is True:
|
| 294 |
+
return data
|
| 295 |
+
|
| 296 |
+
litellm_parent_otel_span: Optional[Any] = None
|
| 297 |
+
# Step 1: Remove the litellm_parent_otel_span
|
| 298 |
+
litellm_parent_otel_span = None
|
| 299 |
+
if isinstance(data, dict):
|
| 300 |
+
# remove litellm_parent_otel_span since this is not picklable
|
| 301 |
+
if "metadata" in data and "litellm_parent_otel_span" in data["metadata"]:
|
| 302 |
+
litellm_parent_otel_span = data["metadata"].pop("litellm_parent_otel_span")
|
| 303 |
+
data["metadata"]["litellm_parent_otel_span"] = "placeholder"
|
| 304 |
+
if (
|
| 305 |
+
"litellm_metadata" in data
|
| 306 |
+
and "litellm_parent_otel_span" in data["litellm_metadata"]
|
| 307 |
+
):
|
| 308 |
+
litellm_parent_otel_span = data["litellm_metadata"].pop(
|
| 309 |
+
"litellm_parent_otel_span"
|
| 310 |
+
)
|
| 311 |
+
data["litellm_metadata"]["litellm_parent_otel_span"] = "placeholder"
|
| 312 |
+
|
| 313 |
+
# Step 2: Per-key deepcopy with fallback
|
| 314 |
+
if isinstance(data, dict):
|
| 315 |
+
new_data = {}
|
| 316 |
+
for k, v in data.items():
|
| 317 |
+
try:
|
| 318 |
+
new_data[k] = copy.deepcopy(v)
|
| 319 |
+
except Exception:
|
| 320 |
+
new_data[k] = v
|
| 321 |
+
else:
|
| 322 |
+
try:
|
| 323 |
+
new_data = copy.deepcopy(data)
|
| 324 |
+
except Exception:
|
| 325 |
+
new_data = data
|
| 326 |
+
|
| 327 |
+
# Step 3: re-add the litellm_parent_otel_span after doing a deep copy
|
| 328 |
+
if isinstance(data, dict) and litellm_parent_otel_span is not None:
|
| 329 |
+
if "metadata" in data and "litellm_parent_otel_span" in data["metadata"]:
|
| 330 |
+
data["metadata"]["litellm_parent_otel_span"] = litellm_parent_otel_span
|
| 331 |
+
if (
|
| 332 |
+
"litellm_metadata" in data
|
| 333 |
+
and "litellm_parent_otel_span" in data["litellm_metadata"]
|
| 334 |
+
):
|
| 335 |
+
data["litellm_metadata"][
|
| 336 |
+
"litellm_parent_otel_span"
|
| 337 |
+
] = litellm_parent_otel_span
|
| 338 |
+
return new_data
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
|
| 342 |
+
"""
|
| 343 |
+
Recursively filter out Exception objects and callable objects from dicts/lists.
|
| 344 |
+
|
| 345 |
+
This is a defensive utility to prevent deepcopy failures when exception objects
|
| 346 |
+
are accidentally stored in parameter dictionaries (e.g., optional_params).
|
| 347 |
+
Also filters callable objects (functions) to prevent JSON serialization errors.
|
| 348 |
+
Exceptions and callables should not be stored in params - this function removes them.
|
| 349 |
+
|
| 350 |
+
Args:
|
| 351 |
+
data: The data structure to filter (dict, list, or any other type)
|
| 352 |
+
max_depth: Maximum recursion depth to prevent infinite loops
|
| 353 |
+
|
| 354 |
+
Returns:
|
| 355 |
+
Filtered data structure with Exception and callable objects removed, or None if the
|
| 356 |
+
entire input was an Exception or callable
|
| 357 |
+
"""
|
| 358 |
+
if max_depth <= 0:
|
| 359 |
+
return data
|
| 360 |
+
|
| 361 |
+
# Skip exception objects
|
| 362 |
+
if isinstance(data, Exception):
|
| 363 |
+
return None
|
| 364 |
+
# Skip callable objects (functions, methods, lambdas) but not classes (type objects)
|
| 365 |
+
if callable(data) and not isinstance(data, type):
|
| 366 |
+
return None
|
| 367 |
+
# Skip known non-serializable object types (Logging, Router, etc.)
|
| 368 |
+
obj_type_name = type(data).__name__
|
| 369 |
+
if obj_type_name in ["Logging", "LiteLLMLoggingObj", "Router"]:
|
| 370 |
+
return None
|
| 371 |
+
|
| 372 |
+
if isinstance(data, dict):
|
| 373 |
+
result: dict[str, Any] = {}
|
| 374 |
+
for k, v in data.items():
|
| 375 |
+
# Skip exception and callable values
|
| 376 |
+
if isinstance(v, Exception) or (callable(v) and not isinstance(v, type)):
|
| 377 |
+
continue
|
| 378 |
+
try:
|
| 379 |
+
filtered = filter_exceptions_from_params(v, max_depth - 1)
|
| 380 |
+
if filtered is not None:
|
| 381 |
+
result[k] = filtered
|
| 382 |
+
except Exception:
|
| 383 |
+
# Skip values that cause errors during filtering
|
| 384 |
+
continue
|
| 385 |
+
return result
|
| 386 |
+
elif isinstance(data, list):
|
| 387 |
+
result_list: list[Any] = []
|
| 388 |
+
for item in data:
|
| 389 |
+
# Skip exception and callable items
|
| 390 |
+
if isinstance(item, Exception) or (
|
| 391 |
+
callable(item) and not isinstance(item, type)
|
| 392 |
+
):
|
| 393 |
+
continue
|
| 394 |
+
try:
|
| 395 |
+
filtered = filter_exceptions_from_params(item, max_depth - 1)
|
| 396 |
+
if filtered is not None:
|
| 397 |
+
result_list.append(filtered)
|
| 398 |
+
except Exception:
|
| 399 |
+
# Skip items that cause errors during filtering
|
| 400 |
+
continue
|
| 401 |
+
return result_list
|
| 402 |
+
else:
|
| 403 |
+
return data
|
| 404 |
+
|
| 405 |
+
|
| 406 |
+
def filter_internal_params(
|
| 407 |
+
data: dict, additional_internal_params: Optional[set] = None
|
| 408 |
+
) -> dict:
|
| 409 |
+
"""
|
| 410 |
+
Filter out LiteLLM internal parameters that shouldn't be sent to provider APIs.
|
| 411 |
+
|
| 412 |
+
This removes internal/MCP-related parameters that are used by LiteLLM internally
|
| 413 |
+
but should not be included in API requests to providers.
|
| 414 |
+
|
| 415 |
+
Args:
|
| 416 |
+
data: Dictionary of parameters to filter
|
| 417 |
+
additional_internal_params: Optional set of additional internal parameter names to filter
|
| 418 |
+
|
| 419 |
+
Returns:
|
| 420 |
+
Filtered dictionary with internal parameters removed
|
| 421 |
+
"""
|
| 422 |
+
if not isinstance(data, dict):
|
| 423 |
+
return data
|
| 424 |
+
|
| 425 |
+
# Known internal parameters that should never be sent to provider APIs
|
| 426 |
+
internal_params = {
|
| 427 |
+
"skip_mcp_handler",
|
| 428 |
+
"mcp_handler_context",
|
| 429 |
+
"_skip_mcp_handler",
|
| 430 |
+
}
|
| 431 |
+
|
| 432 |
+
# Add any additional internal params if provided
|
| 433 |
+
if additional_internal_params:
|
| 434 |
+
internal_params.update(additional_internal_params)
|
| 435 |
+
|
| 436 |
+
# Filter out internal parameters
|
| 437 |
+
return {k: v for k, v in data.items() if k not in internal_params}
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/coroutine_checker.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# CoroutineChecker utility for checking if functions/callables are coroutines or coroutine functions
|
| 2 |
+
|
| 3 |
+
import inspect
|
| 4 |
+
from typing import Any
|
| 5 |
+
from weakref import WeakKeyDictionary
|
| 6 |
+
from litellm.constants import (
|
| 7 |
+
COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY,
|
| 8 |
+
)
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class CoroutineChecker:
|
| 12 |
+
"""Utility class for checking coroutine status of functions and callables.
|
| 13 |
+
|
| 14 |
+
Simple bounded cache using WeakKeyDictionary to avoid memory leaks.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
def __init__(self):
|
| 18 |
+
self._cache = WeakKeyDictionary()
|
| 19 |
+
self._max_size = COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY
|
| 20 |
+
|
| 21 |
+
def is_async_callable(self, callback: Any) -> bool:
|
| 22 |
+
"""Fast, cached check for whether a callback is an async function.
|
| 23 |
+
Falls back gracefully if the object cannot be weak-referenced or cached.
|
| 24 |
+
2.59x speedup.
|
| 25 |
+
"""
|
| 26 |
+
# Fast path: check cache first (most common case)
|
| 27 |
+
try:
|
| 28 |
+
cached = self._cache.get(callback)
|
| 29 |
+
if cached is not None:
|
| 30 |
+
return cached
|
| 31 |
+
except Exception:
|
| 32 |
+
pass
|
| 33 |
+
|
| 34 |
+
# Determine target - optimized path for common cases
|
| 35 |
+
target = callback
|
| 36 |
+
if not inspect.isfunction(target) and not inspect.ismethod(target):
|
| 37 |
+
try:
|
| 38 |
+
call_attr = getattr(target, "__call__", None)
|
| 39 |
+
if call_attr is not None:
|
| 40 |
+
target = call_attr
|
| 41 |
+
except Exception:
|
| 42 |
+
pass
|
| 43 |
+
|
| 44 |
+
# Compute result
|
| 45 |
+
try:
|
| 46 |
+
result = inspect.iscoroutinefunction(target)
|
| 47 |
+
except Exception:
|
| 48 |
+
result = False
|
| 49 |
+
|
| 50 |
+
# Cache the result with size enforcement
|
| 51 |
+
try:
|
| 52 |
+
# Simple size enforcement: clear cache if it gets too large
|
| 53 |
+
if len(self._cache) >= self._max_size:
|
| 54 |
+
self._cache.clear()
|
| 55 |
+
|
| 56 |
+
self._cache[callback] = result
|
| 57 |
+
except Exception:
|
| 58 |
+
pass
|
| 59 |
+
|
| 60 |
+
return result
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
# Global instance for backward compatibility and convenience
|
| 64 |
+
coroutine_checker = CoroutineChecker()
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/credential_accessor.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Utils for accessing credentials."""
|
| 2 |
+
|
| 3 |
+
from typing import List
|
| 4 |
+
|
| 5 |
+
import litellm
|
| 6 |
+
from litellm.types.utils import CredentialItem
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class CredentialAccessor:
|
| 10 |
+
@staticmethod
|
| 11 |
+
def get_credential_values(credential_name: str) -> dict:
|
| 12 |
+
"""Safe accessor for credentials."""
|
| 13 |
+
|
| 14 |
+
if not litellm.credential_list:
|
| 15 |
+
return {}
|
| 16 |
+
for credential in litellm.credential_list:
|
| 17 |
+
if credential.credential_name == credential_name:
|
| 18 |
+
return credential.credential_values.copy()
|
| 19 |
+
return {}
|
| 20 |
+
|
| 21 |
+
@staticmethod
|
| 22 |
+
def upsert_credentials(credentials: List[CredentialItem]):
|
| 23 |
+
"""Add a credential to the list of credentials."""
|
| 24 |
+
|
| 25 |
+
credential_names = [cred.credential_name for cred in litellm.credential_list]
|
| 26 |
+
|
| 27 |
+
for credential in credentials:
|
| 28 |
+
if credential.credential_name in credential_names:
|
| 29 |
+
# Find and replace the existing credential in the list
|
| 30 |
+
for i, existing_cred in enumerate(litellm.credential_list):
|
| 31 |
+
if existing_cred.credential_name == credential.credential_name:
|
| 32 |
+
litellm.credential_list[i] = credential
|
| 33 |
+
break
|
| 34 |
+
else:
|
| 35 |
+
litellm.credential_list.append(credential)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/custom_logger_registry.py
ADDED
|
@@ -0,0 +1,184 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Registry mapping the callback class string to the class type.
|
| 3 |
+
|
| 4 |
+
This is used to get the class type from the callback class string.
|
| 5 |
+
|
| 6 |
+
Example:
|
| 7 |
+
"datadog" -> DataDogLogger
|
| 8 |
+
"prometheus" -> PrometheusLogger
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
from typing import Union
|
| 12 |
+
|
| 13 |
+
from litellm import _custom_logger_compatible_callbacks_literal
|
| 14 |
+
from litellm.integrations.agentops import AgentOps
|
| 15 |
+
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
|
| 16 |
+
from litellm.integrations.argilla import ArgillaLogger
|
| 17 |
+
from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger
|
| 18 |
+
from litellm.integrations.bitbucket import BitBucketPromptManager
|
| 19 |
+
from litellm.integrations.braintrust_logging import BraintrustLogger
|
| 20 |
+
from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
|
| 21 |
+
from litellm.integrations.datadog.datadog import DataDogLogger
|
| 22 |
+
from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger
|
| 23 |
+
from litellm.integrations.datadog.datadog_metrics import DatadogMetricsLogger
|
| 24 |
+
from litellm.integrations.deepeval import DeepEvalLogger
|
| 25 |
+
from litellm.integrations.dotprompt import DotpromptManager
|
| 26 |
+
from litellm.integrations.focus.focus_logger import FocusLogger
|
| 27 |
+
from litellm.integrations.vantage.vantage_logger import VantageLogger
|
| 28 |
+
from litellm.integrations.galileo import GalileoObserve
|
| 29 |
+
from litellm.integrations.gcs_bucket.gcs_bucket import GCSBucketLogger
|
| 30 |
+
from litellm.integrations.gcs_pubsub.pub_sub import GcsPubSubLogger
|
| 31 |
+
from litellm.integrations.gitlab import GitLabPromptManager
|
| 32 |
+
from litellm.integrations.humanloop import HumanloopLogger
|
| 33 |
+
from litellm.integrations.lago import LagoLogger
|
| 34 |
+
from litellm.integrations.langfuse.langfuse_prompt_management import (
|
| 35 |
+
LangfusePromptManagement,
|
| 36 |
+
)
|
| 37 |
+
from litellm.integrations.langsmith import LangsmithLogger
|
| 38 |
+
from litellm.integrations.litellm_agent import LiteLLMAgentModelResolver
|
| 39 |
+
from litellm.integrations.literal_ai import LiteralAILogger
|
| 40 |
+
from litellm.integrations.mlflow import MlflowLogger
|
| 41 |
+
from litellm.integrations.openmeter import OpenMeterLogger
|
| 42 |
+
from litellm.integrations.opentelemetry import OpenTelemetry
|
| 43 |
+
from litellm.integrations.opik.opik import OpikLogger
|
| 44 |
+
from litellm.integrations.posthog import PostHogLogger
|
| 45 |
+
from litellm.integrations.prometheus import PrometheusLogger
|
| 46 |
+
from litellm.integrations.s3_v2 import S3Logger
|
| 47 |
+
from litellm.integrations.sqs import SQSLogger
|
| 48 |
+
from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import (
|
| 49 |
+
VectorStorePreCallHook,
|
| 50 |
+
)
|
| 51 |
+
from litellm.proxy.hooks.dynamic_rate_limiter import _PROXY_DynamicRateLimitHandler
|
| 52 |
+
from litellm.proxy.hooks.dynamic_rate_limiter_v3 import _PROXY_DynamicRateLimitHandlerV3
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class CustomLoggerRegistry:
|
| 56 |
+
"""
|
| 57 |
+
Registry mapping the callback class string to the class type.
|
| 58 |
+
"""
|
| 59 |
+
|
| 60 |
+
CALLBACK_CLASS_STR_TO_CLASS_TYPE = {
|
| 61 |
+
"lago": LagoLogger,
|
| 62 |
+
"openmeter": OpenMeterLogger,
|
| 63 |
+
"braintrust": BraintrustLogger,
|
| 64 |
+
"galileo": GalileoObserve,
|
| 65 |
+
"langsmith": LangsmithLogger,
|
| 66 |
+
"literalai": LiteralAILogger,
|
| 67 |
+
"litellm_agent": LiteLLMAgentModelResolver,
|
| 68 |
+
"prometheus": PrometheusLogger,
|
| 69 |
+
"datadog": DataDogLogger,
|
| 70 |
+
"datadog_llm_observability": DataDogLLMObsLogger,
|
| 71 |
+
"datadog_metrics": DatadogMetricsLogger,
|
| 72 |
+
"gcs_bucket": GCSBucketLogger,
|
| 73 |
+
"opik": OpikLogger,
|
| 74 |
+
"argilla": ArgillaLogger,
|
| 75 |
+
"opentelemetry": OpenTelemetry,
|
| 76 |
+
"azure_storage": AzureBlobStorageLogger,
|
| 77 |
+
"humanloop": HumanloopLogger,
|
| 78 |
+
# OTEL compatible loggers
|
| 79 |
+
"logfire": OpenTelemetry,
|
| 80 |
+
"arize": OpenTelemetry,
|
| 81 |
+
"langfuse_otel": OpenTelemetry,
|
| 82 |
+
"arize_phoenix": OpenTelemetry,
|
| 83 |
+
"langtrace": OpenTelemetry,
|
| 84 |
+
"weave_otel": OpenTelemetry,
|
| 85 |
+
"levo": OpenTelemetry,
|
| 86 |
+
"mlflow": MlflowLogger,
|
| 87 |
+
"langfuse": LangfusePromptManagement,
|
| 88 |
+
"otel": OpenTelemetry,
|
| 89 |
+
"gcs_pubsub": GcsPubSubLogger,
|
| 90 |
+
"anthropic_cache_control_hook": AnthropicCacheControlHook,
|
| 91 |
+
"agentops": AgentOps,
|
| 92 |
+
"deepeval": DeepEvalLogger,
|
| 93 |
+
"s3_v2": S3Logger,
|
| 94 |
+
"aws_sqs": SQSLogger,
|
| 95 |
+
"dynamic_rate_limiter": _PROXY_DynamicRateLimitHandler,
|
| 96 |
+
"dynamic_rate_limiter_v3": _PROXY_DynamicRateLimitHandlerV3,
|
| 97 |
+
"vector_store_pre_call_hook": VectorStorePreCallHook,
|
| 98 |
+
"dotprompt": DotpromptManager,
|
| 99 |
+
"bitbucket": BitBucketPromptManager,
|
| 100 |
+
"gitlab": GitLabPromptManager,
|
| 101 |
+
"cloudzero": CloudZeroLogger,
|
| 102 |
+
"focus": FocusLogger,
|
| 103 |
+
"vantage": VantageLogger,
|
| 104 |
+
"posthog": PostHogLogger,
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
try:
|
| 108 |
+
from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import (
|
| 109 |
+
PagerDutyAlerting,
|
| 110 |
+
)
|
| 111 |
+
from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import (
|
| 112 |
+
ResendEmailLogger,
|
| 113 |
+
)
|
| 114 |
+
from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import (
|
| 115 |
+
SendGridEmailLogger,
|
| 116 |
+
)
|
| 117 |
+
from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import (
|
| 118 |
+
SMTPEmailLogger,
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
from litellm.integrations.generic_api.generic_api_callback import (
|
| 122 |
+
GenericAPILogger,
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
enterprise_loggers = {
|
| 126 |
+
"pagerduty": PagerDutyAlerting,
|
| 127 |
+
"generic_api": GenericAPILogger,
|
| 128 |
+
"resend_email": ResendEmailLogger,
|
| 129 |
+
"sendgrid_email": SendGridEmailLogger,
|
| 130 |
+
"smtp_email": SMTPEmailLogger,
|
| 131 |
+
}
|
| 132 |
+
CALLBACK_CLASS_STR_TO_CLASS_TYPE.update(enterprise_loggers)
|
| 133 |
+
except ImportError:
|
| 134 |
+
pass # enterprise not installed
|
| 135 |
+
|
| 136 |
+
@classmethod
|
| 137 |
+
def get_callback_str_from_class_type(cls, class_type: type) -> Union[str, None]:
|
| 138 |
+
"""
|
| 139 |
+
Get the callback string from the class type.
|
| 140 |
+
|
| 141 |
+
Args:
|
| 142 |
+
class_type: The class type to find the string for
|
| 143 |
+
|
| 144 |
+
Returns:
|
| 145 |
+
str: The callback string, or None if not found
|
| 146 |
+
"""
|
| 147 |
+
for (
|
| 148 |
+
callback_str,
|
| 149 |
+
callback_class,
|
| 150 |
+
) in cls.CALLBACK_CLASS_STR_TO_CLASS_TYPE.items():
|
| 151 |
+
if callback_class == class_type:
|
| 152 |
+
return callback_str
|
| 153 |
+
return None
|
| 154 |
+
|
| 155 |
+
@classmethod
|
| 156 |
+
def get_all_callback_strs_from_class_type(cls, class_type: type) -> list[str]:
|
| 157 |
+
"""
|
| 158 |
+
Get all callback strings that map to the same class type.
|
| 159 |
+
Some class types (like OpenTelemetry) have multiple string mappings.
|
| 160 |
+
|
| 161 |
+
Args:
|
| 162 |
+
class_type: The class type to find all strings for
|
| 163 |
+
|
| 164 |
+
Returns:
|
| 165 |
+
list: List of callback strings that map to the class type
|
| 166 |
+
"""
|
| 167 |
+
callback_strs: list[str] = []
|
| 168 |
+
for (
|
| 169 |
+
callback_str,
|
| 170 |
+
callback_class,
|
| 171 |
+
) in cls.CALLBACK_CLASS_STR_TO_CLASS_TYPE.items():
|
| 172 |
+
if callback_class == class_type:
|
| 173 |
+
callback_strs.append(callback_str)
|
| 174 |
+
return callback_strs
|
| 175 |
+
|
| 176 |
+
@classmethod
|
| 177 |
+
def get_class_type_for_custom_logger_name(
|
| 178 |
+
cls,
|
| 179 |
+
custom_logger_name: _custom_logger_compatible_callbacks_literal,
|
| 180 |
+
) -> type:
|
| 181 |
+
"""
|
| 182 |
+
Get the class type for a given custom logger name
|
| 183 |
+
"""
|
| 184 |
+
return cls.CALLBACK_CLASS_STR_TO_CLASS_TYPE[custom_logger_name]
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/dd_tracing.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Handles Tracing on DataDog Traces.
|
| 3 |
+
|
| 4 |
+
If the ddtrace package is not installed, the tracer will be a no-op.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from contextlib import contextmanager
|
| 8 |
+
from typing import TYPE_CHECKING, Any, Optional, Union
|
| 9 |
+
|
| 10 |
+
from litellm.secret_managers.main import get_secret_bool
|
| 11 |
+
|
| 12 |
+
if TYPE_CHECKING:
|
| 13 |
+
from ddtrace.tracer import Tracer as DD_TRACER
|
| 14 |
+
else:
|
| 15 |
+
DD_TRACER = Any
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class NullSpan:
|
| 19 |
+
"""A no-op span implementation."""
|
| 20 |
+
|
| 21 |
+
def __enter__(self):
|
| 22 |
+
return self
|
| 23 |
+
|
| 24 |
+
def __exit__(self, *args):
|
| 25 |
+
pass
|
| 26 |
+
|
| 27 |
+
def finish(self):
|
| 28 |
+
pass
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@contextmanager
|
| 32 |
+
def null_tracer(name, **kwargs):
|
| 33 |
+
"""Context manager that yields a no-op span."""
|
| 34 |
+
yield NullSpan()
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class NullTracer:
|
| 38 |
+
"""A no-op tracer implementation."""
|
| 39 |
+
|
| 40 |
+
def trace(self, name, **kwargs):
|
| 41 |
+
return NullSpan()
|
| 42 |
+
|
| 43 |
+
def wrap(self, name=None, **kwargs):
|
| 44 |
+
# If called with no arguments (as @tracer.wrap())
|
| 45 |
+
if callable(name):
|
| 46 |
+
return name
|
| 47 |
+
|
| 48 |
+
# If called with arguments (as @tracer.wrap(name="something"))
|
| 49 |
+
def decorator(f):
|
| 50 |
+
return f
|
| 51 |
+
|
| 52 |
+
return decorator
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _should_use_dd_tracer():
|
| 56 |
+
"""Returns True if `USE_DDTRACE` is set to True in .env"""
|
| 57 |
+
return get_secret_bool("USE_DDTRACE", False) is True
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _should_use_dd_profiler():
|
| 61 |
+
"""Returns True if `USE_DDPROFILER` is set to True in .env"""
|
| 62 |
+
return get_secret_bool("USE_DDPROFILER", False) is True
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
# Initialize tracer
|
| 66 |
+
should_use_dd_tracer = _should_use_dd_tracer()
|
| 67 |
+
tracer: Union[NullTracer, DD_TRACER] = NullTracer()
|
| 68 |
+
# We need to ensure tracer is never None and always has the required methods
|
| 69 |
+
if should_use_dd_tracer:
|
| 70 |
+
try:
|
| 71 |
+
from ddtrace import tracer as dd_tracer
|
| 72 |
+
|
| 73 |
+
# Define the type to match what's expected by the code using this module
|
| 74 |
+
tracer = dd_tracer
|
| 75 |
+
except ImportError:
|
| 76 |
+
tracer = NullTracer()
|
| 77 |
+
else:
|
| 78 |
+
tracer = NullTracer()
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def get_active_span() -> Optional[Any]:
|
| 82 |
+
"""
|
| 83 |
+
Return the active Datadog span, checking current span first and then root span.
|
| 84 |
+
"""
|
| 85 |
+
try:
|
| 86 |
+
current_span_fn = getattr(tracer, "current_span", None)
|
| 87 |
+
if callable(current_span_fn):
|
| 88 |
+
current_span = current_span_fn()
|
| 89 |
+
if current_span is not None:
|
| 90 |
+
return current_span
|
| 91 |
+
|
| 92 |
+
current_root_span_fn = getattr(tracer, "current_root_span", None)
|
| 93 |
+
if callable(current_root_span_fn):
|
| 94 |
+
return current_root_span_fn()
|
| 95 |
+
except Exception:
|
| 96 |
+
return None
|
| 97 |
+
return None
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def set_active_span_tag(tag_key: str, tag_value: str) -> bool:
|
| 101 |
+
"""
|
| 102 |
+
Best-effort helper to set a tag on the active Datadog span.
|
| 103 |
+
|
| 104 |
+
Returns:
|
| 105 |
+
bool: True if a span tag was set, False otherwise.
|
| 106 |
+
"""
|
| 107 |
+
if not tag_key or tag_value is None:
|
| 108 |
+
return False
|
| 109 |
+
|
| 110 |
+
span = get_active_span()
|
| 111 |
+
if span is None:
|
| 112 |
+
return False
|
| 113 |
+
|
| 114 |
+
try:
|
| 115 |
+
if hasattr(span, "set_tag_str"):
|
| 116 |
+
span.set_tag_str(tag_key, str(tag_value))
|
| 117 |
+
return True
|
| 118 |
+
if hasattr(span, "set_tag"):
|
| 119 |
+
span.set_tag(tag_key, str(tag_value))
|
| 120 |
+
return True
|
| 121 |
+
except Exception:
|
| 122 |
+
return False
|
| 123 |
+
return False
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/default_encoding.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
|
| 3 |
+
import litellm
|
| 4 |
+
|
| 5 |
+
try:
|
| 6 |
+
# New and recommended way to access resources
|
| 7 |
+
from importlib import resources
|
| 8 |
+
|
| 9 |
+
filename = str(resources.files(litellm).joinpath("litellm_core_utils/tokenizers"))
|
| 10 |
+
except (ImportError, AttributeError):
|
| 11 |
+
# Old way to access resources, which setuptools deprecated some time ago
|
| 12 |
+
import pkg_resources # type: ignore
|
| 13 |
+
|
| 14 |
+
filename = pkg_resources.resource_filename(
|
| 15 |
+
__name__, "litellm_core_utils/tokenizers"
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
# Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory
|
| 19 |
+
# unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR.
|
| 20 |
+
# This keeps tiktoken fully offline-capable by default (see #1071).
|
| 21 |
+
custom_cache_dir = os.getenv("CUSTOM_TIKTOKEN_CACHE_DIR")
|
| 22 |
+
if custom_cache_dir:
|
| 23 |
+
# If the user opts into a custom cache dir, ensure it exists.
|
| 24 |
+
os.makedirs(custom_cache_dir, exist_ok=True)
|
| 25 |
+
cache_dir = custom_cache_dir
|
| 26 |
+
else:
|
| 27 |
+
cache_dir = filename
|
| 28 |
+
|
| 29 |
+
os.environ[
|
| 30 |
+
"TIKTOKEN_CACHE_DIR"
|
| 31 |
+
] = cache_dir # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071
|
| 32 |
+
|
| 33 |
+
import tiktoken
|
| 34 |
+
import time
|
| 35 |
+
import random
|
| 36 |
+
|
| 37 |
+
# Retry logic to handle race conditions when multiple processes try to create
|
| 38 |
+
# the tiktoken cache file simultaneously (common in parallel test execution on Windows)
|
| 39 |
+
_max_retries = 5
|
| 40 |
+
_retry_delay = 0.1 # Start with 100ms
|
| 41 |
+
|
| 42 |
+
for attempt in range(_max_retries):
|
| 43 |
+
try:
|
| 44 |
+
encoding = tiktoken.get_encoding("cl100k_base")
|
| 45 |
+
break
|
| 46 |
+
except (FileExistsError, OSError):
|
| 47 |
+
if attempt == _max_retries - 1:
|
| 48 |
+
# Last attempt, re-raise the exception
|
| 49 |
+
raise
|
| 50 |
+
# Exponential backoff with jitter to reduce collision probability
|
| 51 |
+
delay = _retry_delay * (2**attempt) + random.uniform(0, 0.1)
|
| 52 |
+
time.sleep(delay)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/dot_notation_indexing.py
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Path-based navigation utilities for nested dictionaries.
|
| 3 |
+
|
| 4 |
+
This module provides utilities for reading and deleting values in nested
|
| 5 |
+
dictionaries using dot notation and JSONPath-like array syntax.
|
| 6 |
+
|
| 7 |
+
Custom implementation with zero external dependencies.
|
| 8 |
+
|
| 9 |
+
Supported syntax:
|
| 10 |
+
- "field" - top-level field
|
| 11 |
+
- "parent.child" - nested field
|
| 12 |
+
- "parent\\.with\\.dots.child" - keys containing dots (escape with backslash)
|
| 13 |
+
- "array[*]" - all array elements (wildcard)
|
| 14 |
+
- "array[0]" - specific array element (index)
|
| 15 |
+
- "array[*].field" - field in all array elements
|
| 16 |
+
|
| 17 |
+
Examples:
|
| 18 |
+
>>> data = {"tools": [{"name": "t1", "input_examples": ["ex"]}]}
|
| 19 |
+
>>> delete_nested_value(data, "tools[*].input_examples")
|
| 20 |
+
{"tools": [{"name": "t1"}]}
|
| 21 |
+
|
| 22 |
+
Used by JWT Auth to get the user role from the token, and by
|
| 23 |
+
additional_drop_params to remove nested fields from optional parameters.
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
from typing import Any, Dict, List, Optional, TypeVar, Union
|
| 27 |
+
|
| 28 |
+
T = TypeVar("T")
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def get_nested_value(
|
| 32 |
+
data: Dict[str, Any], key_path: str, default: Optional[T] = None
|
| 33 |
+
) -> Optional[T]:
|
| 34 |
+
"""
|
| 35 |
+
Retrieves a value from a nested dictionary using dot notation.
|
| 36 |
+
|
| 37 |
+
Args:
|
| 38 |
+
data: The dictionary to search in
|
| 39 |
+
key_path: The path to the value using dot notation (e.g., "a.b.c")
|
| 40 |
+
default: The default value to return if the path is not found
|
| 41 |
+
|
| 42 |
+
Returns:
|
| 43 |
+
The value at the specified path, or the default value if not found
|
| 44 |
+
|
| 45 |
+
Example:
|
| 46 |
+
>>> data = {"a": {"b": {"c": "value"}}}
|
| 47 |
+
>>> get_nested_value(data, "a.b.c")
|
| 48 |
+
'value'
|
| 49 |
+
>>> get_nested_value(data, "a.b.d", "default")
|
| 50 |
+
'default'
|
| 51 |
+
>>> data = {"kubernetes.io": {"namespace": "default"}}
|
| 52 |
+
>>> get_nested_value(data, "kubernetes\\.io.namespace")
|
| 53 |
+
'default'
|
| 54 |
+
"""
|
| 55 |
+
if not key_path:
|
| 56 |
+
return default
|
| 57 |
+
|
| 58 |
+
# Remove metadata. prefix if it exists
|
| 59 |
+
key_path = (
|
| 60 |
+
key_path.replace("metadata.", "", 1)
|
| 61 |
+
if key_path.startswith("metadata.")
|
| 62 |
+
else key_path
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
# Split the key path into parts, respecting escaped dots (\.)
|
| 66 |
+
# Use a temporary placeholder, split on unescaped dots, then restore
|
| 67 |
+
placeholder = "\x00"
|
| 68 |
+
parts = key_path.replace("\\.", placeholder).split(".")
|
| 69 |
+
parts = [p.replace(placeholder, ".") for p in parts]
|
| 70 |
+
|
| 71 |
+
# Traverse through the dictionary
|
| 72 |
+
current: Any = data
|
| 73 |
+
for part in parts:
|
| 74 |
+
try:
|
| 75 |
+
current = current[part]
|
| 76 |
+
except (KeyError, TypeError):
|
| 77 |
+
return default
|
| 78 |
+
|
| 79 |
+
# If default is None, we can return any type
|
| 80 |
+
if default is None:
|
| 81 |
+
return current
|
| 82 |
+
|
| 83 |
+
# Otherwise, ensure the type matches the default
|
| 84 |
+
return current if isinstance(current, type(default)) else default
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _parse_path_segments(path: str) -> list:
|
| 88 |
+
"""
|
| 89 |
+
Parse a JSONPath-like string into segments using regex.
|
| 90 |
+
|
| 91 |
+
Handles:
|
| 92 |
+
- Dot notation: "a.b.c" → ["a", "b", "c"]
|
| 93 |
+
- Array wildcards: "a[*].b" → ["a", "[*]", "b"]
|
| 94 |
+
- Array indices: "a[0].b" → ["a", "[0]", "b"]
|
| 95 |
+
|
| 96 |
+
Args:
|
| 97 |
+
path: JSONPath-like path string
|
| 98 |
+
|
| 99 |
+
Returns:
|
| 100 |
+
List of path segments
|
| 101 |
+
|
| 102 |
+
Example:
|
| 103 |
+
>>> _parse_path_segments("tools[*].arr[0].field")
|
| 104 |
+
["tools", "[*]", "arr", "[0]", "field"]
|
| 105 |
+
"""
|
| 106 |
+
import re
|
| 107 |
+
|
| 108 |
+
# Match field names OR bracket expressions
|
| 109 |
+
# Pattern: field_name (anything except . or [) | [anything_in_brackets]
|
| 110 |
+
pattern = r"[^\.\[]+|\[[^\]]*\]"
|
| 111 |
+
segments = re.findall(pattern, path)
|
| 112 |
+
return segments
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _delete_nested_value_custom(
|
| 116 |
+
data: Union[Dict[str, Any], List[Any]],
|
| 117 |
+
segments: list,
|
| 118 |
+
segment_index: int = 0,
|
| 119 |
+
) -> None:
|
| 120 |
+
"""
|
| 121 |
+
Recursively delete a field from nested data using parsed segments.
|
| 122 |
+
|
| 123 |
+
Modifies data in-place (caller must deep copy first).
|
| 124 |
+
|
| 125 |
+
Args:
|
| 126 |
+
data: Dictionary or list to modify
|
| 127 |
+
segments: Parsed path segments
|
| 128 |
+
segment_index: Current position in segments list
|
| 129 |
+
"""
|
| 130 |
+
if segment_index >= len(segments):
|
| 131 |
+
return
|
| 132 |
+
|
| 133 |
+
segment = segments[segment_index]
|
| 134 |
+
is_last = segment_index == len(segments) - 1
|
| 135 |
+
|
| 136 |
+
# Handle array wildcard: [*]
|
| 137 |
+
if segment == "[*]":
|
| 138 |
+
if isinstance(data, list):
|
| 139 |
+
for item in data:
|
| 140 |
+
if is_last:
|
| 141 |
+
# Can't delete array elements themselves, skip
|
| 142 |
+
pass
|
| 143 |
+
else:
|
| 144 |
+
# Only recurse if item is a dict or list (nested structure)
|
| 145 |
+
if isinstance(item, (dict, list)):
|
| 146 |
+
_delete_nested_value_custom(item, segments, segment_index + 1)
|
| 147 |
+
return
|
| 148 |
+
|
| 149 |
+
# Handle array index: [0], [1], [2], etc.
|
| 150 |
+
if segment.startswith("[") and segment.endswith("]"):
|
| 151 |
+
try:
|
| 152 |
+
index = int(segment[1:-1])
|
| 153 |
+
if isinstance(data, list) and 0 <= index < len(data):
|
| 154 |
+
if is_last:
|
| 155 |
+
# Can't delete array elements themselves, skip
|
| 156 |
+
pass
|
| 157 |
+
else:
|
| 158 |
+
# Only recurse if element is a dict or list (nested structure)
|
| 159 |
+
element = data[index]
|
| 160 |
+
if isinstance(element, (dict, list)):
|
| 161 |
+
_delete_nested_value_custom(
|
| 162 |
+
element, segments, segment_index + 1
|
| 163 |
+
)
|
| 164 |
+
except (ValueError, IndexError):
|
| 165 |
+
# Invalid index, skip
|
| 166 |
+
pass
|
| 167 |
+
return
|
| 168 |
+
|
| 169 |
+
# Handle regular field navigation
|
| 170 |
+
if isinstance(data, dict):
|
| 171 |
+
if is_last:
|
| 172 |
+
# Delete the field
|
| 173 |
+
data.pop(segment, None)
|
| 174 |
+
else:
|
| 175 |
+
# Navigate deeper
|
| 176 |
+
if segment in data:
|
| 177 |
+
next_segment = (
|
| 178 |
+
segments[segment_index + 1]
|
| 179 |
+
if segment_index + 1 < len(segments)
|
| 180 |
+
else None
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
# If next segment is array notation, current field should be list
|
| 184 |
+
if next_segment and (next_segment.startswith("[")):
|
| 185 |
+
if isinstance(data[segment], list):
|
| 186 |
+
_delete_nested_value_custom(
|
| 187 |
+
data[segment], segments, segment_index + 1
|
| 188 |
+
)
|
| 189 |
+
# Otherwise navigate into dict
|
| 190 |
+
elif isinstance(data[segment], dict):
|
| 191 |
+
_delete_nested_value_custom(
|
| 192 |
+
data[segment], segments, segment_index + 1
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def delete_nested_value(
|
| 197 |
+
data: Dict[str, Any],
|
| 198 |
+
path: str,
|
| 199 |
+
depth: int = 0,
|
| 200 |
+
max_depth: int = 20,
|
| 201 |
+
) -> Dict[str, Any]:
|
| 202 |
+
"""
|
| 203 |
+
Delete a field from nested data using JSONPath notation.
|
| 204 |
+
|
| 205 |
+
Custom implementation - no external dependencies.
|
| 206 |
+
|
| 207 |
+
Supports:
|
| 208 |
+
- "field" - top-level field
|
| 209 |
+
- "parent.child" - nested field
|
| 210 |
+
- "array[*]" - all array elements (wildcard)
|
| 211 |
+
- "array[0]" - specific array element (index)
|
| 212 |
+
- "array[*].field" - field in all array elements
|
| 213 |
+
|
| 214 |
+
Args:
|
| 215 |
+
data: Dictionary to modify (creates deep copy)
|
| 216 |
+
path: JSONPath-like path string
|
| 217 |
+
depth: Current recursion depth (kept for API compatibility)
|
| 218 |
+
max_depth: Maximum recursion depth (kept for API compatibility)
|
| 219 |
+
|
| 220 |
+
Returns:
|
| 221 |
+
New dictionary with field removed at path
|
| 222 |
+
|
| 223 |
+
Example:
|
| 224 |
+
>>> data = {"tools": [{"name": "t1", "input_examples": ["ex"]}]}
|
| 225 |
+
>>> delete_nested_value(data, "tools[*].input_examples")
|
| 226 |
+
{"tools": [{"name": "t1"}]}
|
| 227 |
+
"""
|
| 228 |
+
import copy
|
| 229 |
+
|
| 230 |
+
result = copy.deepcopy(data)
|
| 231 |
+
|
| 232 |
+
try:
|
| 233 |
+
# Parse path into segments
|
| 234 |
+
segments = _parse_path_segments(path)
|
| 235 |
+
|
| 236 |
+
if not segments:
|
| 237 |
+
return result
|
| 238 |
+
|
| 239 |
+
# Delete using custom recursive implementation
|
| 240 |
+
_delete_nested_value_custom(result, segments, 0)
|
| 241 |
+
|
| 242 |
+
except Exception:
|
| 243 |
+
# Invalid path or parsing error - silently skip
|
| 244 |
+
pass
|
| 245 |
+
|
| 246 |
+
return result
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def is_nested_path(path: str) -> bool:
|
| 250 |
+
"""
|
| 251 |
+
Check if path requires nested handling.
|
| 252 |
+
|
| 253 |
+
Returns True if path contains '.' or '[' (array notation).
|
| 254 |
+
"""
|
| 255 |
+
return "." in path or "[" in path
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/duration_parser.py
ADDED
|
@@ -0,0 +1,391 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Helper utilities for parsing durations - 1s, 1d, 10d, 30d, 1mo, 2mo
|
| 3 |
+
|
| 4 |
+
duration_in_seconds is used in diff parts of the code base, example
|
| 5 |
+
- Router - Provider budget routing
|
| 6 |
+
- Proxy - Key, Team Generation
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import re
|
| 10 |
+
import time
|
| 11 |
+
from datetime import datetime, timedelta, timezone, tzinfo
|
| 12 |
+
from typing import Optional, Tuple
|
| 13 |
+
from zoneinfo import ZoneInfo
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def _extract_from_regex(duration: str) -> Tuple[int, str]:
|
| 17 |
+
match = re.match(r"(\d+)(mo|[smhdw]?)", duration)
|
| 18 |
+
|
| 19 |
+
if not match:
|
| 20 |
+
raise ValueError("Invalid duration format")
|
| 21 |
+
|
| 22 |
+
value, unit = match.groups()
|
| 23 |
+
value = int(value)
|
| 24 |
+
|
| 25 |
+
return value, unit
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def get_last_day_of_month(year, month):
|
| 29 |
+
# Handle December case
|
| 30 |
+
if month == 12:
|
| 31 |
+
return 31
|
| 32 |
+
# Next month is January, so subtract a day from March 1st
|
| 33 |
+
next_month = datetime(year=year, month=month + 1, day=1)
|
| 34 |
+
last_day_of_month = (next_month - timedelta(days=1)).day
|
| 35 |
+
return last_day_of_month
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def duration_in_seconds(duration: str) -> int:
|
| 39 |
+
"""
|
| 40 |
+
Parameters:
|
| 41 |
+
- duration:
|
| 42 |
+
- "<number>s" - seconds
|
| 43 |
+
- "<number>m" - minutes
|
| 44 |
+
- "<number>h" - hours
|
| 45 |
+
- "<number>d" - days
|
| 46 |
+
- "<number>w" - weeks
|
| 47 |
+
- "<number>mo" - months
|
| 48 |
+
|
| 49 |
+
Returns time in seconds till when budget needs to be reset
|
| 50 |
+
"""
|
| 51 |
+
value, unit = _extract_from_regex(duration=duration)
|
| 52 |
+
|
| 53 |
+
if unit == "s":
|
| 54 |
+
return value
|
| 55 |
+
elif unit == "m":
|
| 56 |
+
return value * 60
|
| 57 |
+
elif unit == "h":
|
| 58 |
+
return value * 3600
|
| 59 |
+
elif unit == "d":
|
| 60 |
+
return value * 86400
|
| 61 |
+
elif unit == "w":
|
| 62 |
+
return value * 604800
|
| 63 |
+
elif unit == "mo":
|
| 64 |
+
now = time.time()
|
| 65 |
+
current_time = datetime.fromtimestamp(now)
|
| 66 |
+
|
| 67 |
+
# Calculate target month and year, handling overflow past December
|
| 68 |
+
total_months = current_time.month - 1 + value # 0-indexed months
|
| 69 |
+
target_year = current_time.year + total_months // 12
|
| 70 |
+
target_month = total_months % 12 + 1 # back to 1-indexed
|
| 71 |
+
|
| 72 |
+
# Determine the day to set for next month
|
| 73 |
+
target_day = current_time.day
|
| 74 |
+
last_day_of_target_month = get_last_day_of_month(target_year, target_month)
|
| 75 |
+
|
| 76 |
+
if target_day > last_day_of_target_month:
|
| 77 |
+
target_day = last_day_of_target_month
|
| 78 |
+
|
| 79 |
+
next_month = datetime(
|
| 80 |
+
year=target_year,
|
| 81 |
+
month=target_month,
|
| 82 |
+
day=target_day,
|
| 83 |
+
hour=current_time.hour,
|
| 84 |
+
minute=current_time.minute,
|
| 85 |
+
second=current_time.second,
|
| 86 |
+
microsecond=current_time.microsecond,
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
# Calculate the duration until the first day of the next month
|
| 90 |
+
duration_until_next_month = next_month - current_time
|
| 91 |
+
return int(duration_until_next_month.total_seconds())
|
| 92 |
+
|
| 93 |
+
else:
|
| 94 |
+
raise ValueError(f"Unsupported duration unit, passed duration: {duration}")
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def get_next_standardized_reset_time(
|
| 98 |
+
duration: str, current_time: datetime, timezone_str: str = "UTC"
|
| 99 |
+
) -> datetime:
|
| 100 |
+
"""
|
| 101 |
+
Get the next standardized reset time based on the duration.
|
| 102 |
+
|
| 103 |
+
All durations will reset at predictable intervals, aligned from the current time:
|
| 104 |
+
- Nd: If N=1, reset at next midnight; if N>1, reset every N days from now
|
| 105 |
+
- Nh: Every N hours, aligned to hour boundaries (e.g., 1:00, 2:00)
|
| 106 |
+
- Nm: Every N minutes, aligned to minute boundaries (e.g., 1:05, 1:10)
|
| 107 |
+
- Ns: Every N seconds, aligned to second boundaries
|
| 108 |
+
|
| 109 |
+
Parameters:
|
| 110 |
+
- duration: Duration string (e.g. "30s", "30m", "30h", "30d")
|
| 111 |
+
- current_time: Current datetime
|
| 112 |
+
- timezone_str: Timezone string (e.g. "UTC", "US/Eastern", "Asia/Kolkata")
|
| 113 |
+
|
| 114 |
+
Returns:
|
| 115 |
+
- Next reset time at a standardized interval in the specified timezone
|
| 116 |
+
"""
|
| 117 |
+
# Set up timezone and normalize current time
|
| 118 |
+
current_time, tz = _setup_timezone(current_time, timezone_str)
|
| 119 |
+
|
| 120 |
+
# Parse duration
|
| 121 |
+
value, unit = _parse_duration(duration)
|
| 122 |
+
if value is None:
|
| 123 |
+
# Fall back to default if format is invalid
|
| 124 |
+
return current_time.replace(
|
| 125 |
+
hour=0, minute=0, second=0, microsecond=0
|
| 126 |
+
) + timedelta(days=1)
|
| 127 |
+
|
| 128 |
+
# Midnight of the current day in the specified timezone
|
| 129 |
+
base_midnight = current_time.replace(hour=0, minute=0, second=0, microsecond=0)
|
| 130 |
+
|
| 131 |
+
# Handle different time units
|
| 132 |
+
if unit == "d":
|
| 133 |
+
return _handle_day_reset(current_time, base_midnight, value, tz)
|
| 134 |
+
elif unit == "h":
|
| 135 |
+
return _handle_hour_reset(current_time, base_midnight, value)
|
| 136 |
+
elif unit == "m":
|
| 137 |
+
return _handle_minute_reset(current_time, base_midnight, value)
|
| 138 |
+
elif unit == "s":
|
| 139 |
+
return _handle_second_reset(current_time, base_midnight, value)
|
| 140 |
+
elif unit == "mo":
|
| 141 |
+
return _handle_month_reset(current_time, base_midnight, value)
|
| 142 |
+
else:
|
| 143 |
+
# Unrecognized unit, default to next midnight
|
| 144 |
+
return base_midnight + timedelta(days=1)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _setup_timezone(
|
| 148 |
+
current_time: datetime, timezone_str: str = "UTC"
|
| 149 |
+
) -> Tuple[datetime, tzinfo]:
|
| 150 |
+
"""Set up timezone and normalize current time to that timezone."""
|
| 151 |
+
try:
|
| 152 |
+
if timezone_str is None:
|
| 153 |
+
tz: tzinfo = timezone.utc
|
| 154 |
+
else:
|
| 155 |
+
tz = ZoneInfo(timezone_str)
|
| 156 |
+
except Exception:
|
| 157 |
+
# If timezone is invalid, fall back to UTC
|
| 158 |
+
tz = timezone.utc
|
| 159 |
+
|
| 160 |
+
# Convert current_time to the target timezone
|
| 161 |
+
if current_time.tzinfo is None:
|
| 162 |
+
# Naive datetime - assume it's UTC
|
| 163 |
+
utc_time = current_time.replace(tzinfo=timezone.utc)
|
| 164 |
+
current_time = utc_time.astimezone(tz)
|
| 165 |
+
else:
|
| 166 |
+
# Already has timezone - convert to target timezone
|
| 167 |
+
current_time = current_time.astimezone(tz)
|
| 168 |
+
|
| 169 |
+
return current_time, tz
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _parse_duration(duration: str) -> Tuple[Optional[int], Optional[str]]:
|
| 173 |
+
"""Parse the duration string into value and unit."""
|
| 174 |
+
match = re.match(r"(\d+)([a-z]+)", duration)
|
| 175 |
+
if not match:
|
| 176 |
+
return None, None
|
| 177 |
+
|
| 178 |
+
value, unit = match.groups()
|
| 179 |
+
return int(value), unit
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def _handle_day_reset(
|
| 183 |
+
current_time: datetime, base_midnight: datetime, value: int, tz: tzinfo
|
| 184 |
+
) -> datetime:
|
| 185 |
+
"""Handle day-based reset times."""
|
| 186 |
+
# Handle zero value - immediate expiration
|
| 187 |
+
if value == 0:
|
| 188 |
+
return current_time
|
| 189 |
+
|
| 190 |
+
if value == 1: # Daily reset at midnight
|
| 191 |
+
return base_midnight + timedelta(days=1)
|
| 192 |
+
elif value == 7: # Weekly reset on Monday at midnight
|
| 193 |
+
days_until_monday = (7 - current_time.weekday()) % 7
|
| 194 |
+
if days_until_monday == 0: # If today is Monday
|
| 195 |
+
days_until_monday = 7
|
| 196 |
+
return base_midnight + timedelta(days=days_until_monday)
|
| 197 |
+
elif value == 30: # Monthly reset on 1st at midnight
|
| 198 |
+
# Get 1st of next month at midnight
|
| 199 |
+
if current_time.month == 12:
|
| 200 |
+
next_reset = datetime(
|
| 201 |
+
year=current_time.year + 1,
|
| 202 |
+
month=1,
|
| 203 |
+
day=1,
|
| 204 |
+
hour=0,
|
| 205 |
+
minute=0,
|
| 206 |
+
second=0,
|
| 207 |
+
microsecond=0,
|
| 208 |
+
tzinfo=tz,
|
| 209 |
+
)
|
| 210 |
+
else:
|
| 211 |
+
next_reset = datetime(
|
| 212 |
+
year=current_time.year,
|
| 213 |
+
month=current_time.month + 1,
|
| 214 |
+
day=1,
|
| 215 |
+
hour=0,
|
| 216 |
+
minute=0,
|
| 217 |
+
second=0,
|
| 218 |
+
microsecond=0,
|
| 219 |
+
tzinfo=tz,
|
| 220 |
+
)
|
| 221 |
+
return next_reset
|
| 222 |
+
else: # Custom day value - next interval is value days from current
|
| 223 |
+
return current_time.replace(
|
| 224 |
+
hour=0, minute=0, second=0, microsecond=0
|
| 225 |
+
) + timedelta(days=value)
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def _handle_hour_reset(
|
| 229 |
+
current_time: datetime, base_midnight: datetime, value: int
|
| 230 |
+
) -> datetime:
|
| 231 |
+
"""Handle hour-based reset times."""
|
| 232 |
+
# Handle zero value - immediate expiration
|
| 233 |
+
if value == 0:
|
| 234 |
+
return current_time
|
| 235 |
+
|
| 236 |
+
current_hour = current_time.hour
|
| 237 |
+
current_minute = current_time.minute
|
| 238 |
+
current_second = current_time.second
|
| 239 |
+
current_microsecond = current_time.microsecond
|
| 240 |
+
|
| 241 |
+
# Calculate next hour aligned with the value
|
| 242 |
+
if current_minute == 0 and current_second == 0 and current_microsecond == 0:
|
| 243 |
+
next_hour = (
|
| 244 |
+
current_hour + value - (current_hour % value)
|
| 245 |
+
if current_hour % value != 0
|
| 246 |
+
else current_hour + value
|
| 247 |
+
)
|
| 248 |
+
else:
|
| 249 |
+
next_hour = (
|
| 250 |
+
current_hour + value - (current_hour % value)
|
| 251 |
+
if current_hour % value != 0
|
| 252 |
+
else current_hour + value
|
| 253 |
+
)
|
| 254 |
+
|
| 255 |
+
# Handle overnight case
|
| 256 |
+
if next_hour >= 24:
|
| 257 |
+
next_hour = next_hour % 24
|
| 258 |
+
next_day = base_midnight + timedelta(days=1)
|
| 259 |
+
return next_day.replace(hour=next_hour)
|
| 260 |
+
|
| 261 |
+
return current_time.replace(hour=next_hour, minute=0, second=0, microsecond=0)
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def _handle_minute_reset(
|
| 265 |
+
current_time: datetime, base_midnight: datetime, value: int
|
| 266 |
+
) -> datetime:
|
| 267 |
+
"""Handle minute-based reset times."""
|
| 268 |
+
# Handle zero value - immediate expiration
|
| 269 |
+
if value == 0:
|
| 270 |
+
return current_time
|
| 271 |
+
|
| 272 |
+
current_hour = current_time.hour
|
| 273 |
+
current_minute = current_time.minute
|
| 274 |
+
current_second = current_time.second
|
| 275 |
+
current_microsecond = current_time.microsecond
|
| 276 |
+
|
| 277 |
+
# Calculate next minute aligned with the value
|
| 278 |
+
if current_second == 0 and current_microsecond == 0:
|
| 279 |
+
next_minute = (
|
| 280 |
+
current_minute + value - (current_minute % value)
|
| 281 |
+
if current_minute % value != 0
|
| 282 |
+
else current_minute + value
|
| 283 |
+
)
|
| 284 |
+
else:
|
| 285 |
+
next_minute = (
|
| 286 |
+
current_minute + value - (current_minute % value)
|
| 287 |
+
if current_minute % value != 0
|
| 288 |
+
else current_minute + value
|
| 289 |
+
)
|
| 290 |
+
|
| 291 |
+
# Handle hour rollover
|
| 292 |
+
next_hour = current_hour + (next_minute // 60)
|
| 293 |
+
next_minute = next_minute % 60
|
| 294 |
+
|
| 295 |
+
# Handle overnight case
|
| 296 |
+
if next_hour >= 24:
|
| 297 |
+
next_hour = next_hour % 24
|
| 298 |
+
next_day = base_midnight + timedelta(days=1)
|
| 299 |
+
return next_day.replace(
|
| 300 |
+
hour=next_hour, minute=next_minute, second=0, microsecond=0
|
| 301 |
+
)
|
| 302 |
+
|
| 303 |
+
return current_time.replace(
|
| 304 |
+
hour=next_hour, minute=next_minute, second=0, microsecond=0
|
| 305 |
+
)
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def _handle_second_reset(
|
| 309 |
+
current_time: datetime, base_midnight: datetime, value: int
|
| 310 |
+
) -> datetime:
|
| 311 |
+
"""Handle second-based reset times."""
|
| 312 |
+
# Handle zero value - immediate expiration
|
| 313 |
+
if value == 0:
|
| 314 |
+
return current_time
|
| 315 |
+
|
| 316 |
+
current_hour = current_time.hour
|
| 317 |
+
current_minute = current_time.minute
|
| 318 |
+
current_second = current_time.second
|
| 319 |
+
current_microsecond = current_time.microsecond
|
| 320 |
+
|
| 321 |
+
# Calculate next second aligned with the value
|
| 322 |
+
if current_microsecond == 0:
|
| 323 |
+
next_second = (
|
| 324 |
+
current_second + value - (current_second % value)
|
| 325 |
+
if current_second % value != 0
|
| 326 |
+
else current_second + value
|
| 327 |
+
)
|
| 328 |
+
else:
|
| 329 |
+
next_second = (
|
| 330 |
+
current_second + value - (current_second % value)
|
| 331 |
+
if current_second % value != 0
|
| 332 |
+
else current_second + value
|
| 333 |
+
)
|
| 334 |
+
|
| 335 |
+
# Handle minute rollover
|
| 336 |
+
additional_minutes = next_second // 60
|
| 337 |
+
next_second = next_second % 60
|
| 338 |
+
next_minute = current_minute + additional_minutes
|
| 339 |
+
|
| 340 |
+
# Handle hour rollover
|
| 341 |
+
next_hour = current_hour + (next_minute // 60)
|
| 342 |
+
next_minute = next_minute % 60
|
| 343 |
+
|
| 344 |
+
# Handle overnight case
|
| 345 |
+
if next_hour >= 24:
|
| 346 |
+
next_hour = next_hour % 24
|
| 347 |
+
next_day = base_midnight + timedelta(days=1)
|
| 348 |
+
return next_day.replace(
|
| 349 |
+
hour=next_hour, minute=next_minute, second=next_second, microsecond=0
|
| 350 |
+
)
|
| 351 |
+
|
| 352 |
+
return current_time.replace(
|
| 353 |
+
hour=next_hour, minute=next_minute, second=next_second, microsecond=0
|
| 354 |
+
)
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
def _handle_month_reset(
|
| 358 |
+
current_time: datetime, base_midnight: datetime, value: int
|
| 359 |
+
) -> datetime:
|
| 360 |
+
"""
|
| 361 |
+
Handle monthly reset times. For monthly resets, we always reset at the start of the next month.
|
| 362 |
+
|
| 363 |
+
Args:
|
| 364 |
+
current_time: Current datetime
|
| 365 |
+
base_midnight: Midnight of current day
|
| 366 |
+
value: Number of months (currently only supports 1 month resets)
|
| 367 |
+
|
| 368 |
+
Returns:
|
| 369 |
+
datetime: First day of next month at midnight
|
| 370 |
+
"""
|
| 371 |
+
if value != 1:
|
| 372 |
+
raise ValueError("Monthly resets currently only support 1 month intervals")
|
| 373 |
+
|
| 374 |
+
# Get the first day of next month
|
| 375 |
+
if current_time.month == 12:
|
| 376 |
+
next_month = 1
|
| 377 |
+
next_year = current_time.year + 1
|
| 378 |
+
else:
|
| 379 |
+
next_month = current_time.month + 1
|
| 380 |
+
next_year = current_time.year
|
| 381 |
+
|
| 382 |
+
return datetime(
|
| 383 |
+
year=next_year,
|
| 384 |
+
month=next_month,
|
| 385 |
+
day=1,
|
| 386 |
+
hour=0,
|
| 387 |
+
minute=0,
|
| 388 |
+
second=0,
|
| 389 |
+
microsecond=0,
|
| 390 |
+
tzinfo=current_time.tzinfo,
|
| 391 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/env_utils.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Utility helpers for reading and parsing environment variables.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import os
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def get_env_int(env_var: str, default: int) -> int:
|
| 9 |
+
"""Parse an environment variable as an integer, falling back to default on invalid values.
|
| 10 |
+
|
| 11 |
+
Handles empty strings, whitespace, and non-numeric values gracefully
|
| 12 |
+
so that misconfiguration doesn't crash the process at import time.
|
| 13 |
+
"""
|
| 14 |
+
raw = os.getenv(env_var)
|
| 15 |
+
if raw is None:
|
| 16 |
+
return default
|
| 17 |
+
raw = raw.strip()
|
| 18 |
+
try:
|
| 19 |
+
return int(raw)
|
| 20 |
+
except (ValueError, TypeError):
|
| 21 |
+
return default
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/fallback_utils.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from litellm._uuid import uuid
|
| 2 |
+
from typing import Optional
|
| 3 |
+
|
| 4 |
+
import litellm
|
| 5 |
+
from litellm._logging import verbose_logger
|
| 6 |
+
from litellm.litellm_core_utils.core_helpers import (
|
| 7 |
+
safe_deep_copy,
|
| 8 |
+
filter_internal_params,
|
| 9 |
+
)
|
| 10 |
+
|
| 11 |
+
from .asyncify import run_async_function
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
async def async_completion_with_fallbacks(**kwargs):
|
| 15 |
+
"""
|
| 16 |
+
Asynchronously attempts completion with fallback models if the primary model fails.
|
| 17 |
+
|
| 18 |
+
Args:
|
| 19 |
+
**kwargs: Keyword arguments for completion, including:
|
| 20 |
+
- model (str): Primary model to use
|
| 21 |
+
- fallbacks (List[Union[str, dict]]): List of fallback models/configs
|
| 22 |
+
- Other completion parameters
|
| 23 |
+
|
| 24 |
+
Returns:
|
| 25 |
+
ModelResponse: The completion response from the first successful model
|
| 26 |
+
|
| 27 |
+
Raises:
|
| 28 |
+
Exception: If all models fail and no response is generated
|
| 29 |
+
"""
|
| 30 |
+
# Extract and prepare parameters
|
| 31 |
+
nested_kwargs = kwargs.pop("kwargs", {})
|
| 32 |
+
original_model = kwargs["model"]
|
| 33 |
+
model = original_model
|
| 34 |
+
fallbacks = [original_model] + nested_kwargs.pop("fallbacks", [])
|
| 35 |
+
kwargs.pop("acompletion", None) # Remove to prevent keyword conflicts
|
| 36 |
+
litellm_call_id = str(uuid.uuid4())
|
| 37 |
+
base_kwargs = {**kwargs, **nested_kwargs, "litellm_call_id": litellm_call_id}
|
| 38 |
+
|
| 39 |
+
# fields to remove
|
| 40 |
+
base_kwargs.pop("model", None) # Remove model as it will be set per fallback
|
| 41 |
+
litellm_logging_obj = base_kwargs.pop("litellm_logging_obj", None)
|
| 42 |
+
|
| 43 |
+
# Try each fallback model
|
| 44 |
+
most_recent_exception_str: Optional[str] = None
|
| 45 |
+
for fallback in fallbacks:
|
| 46 |
+
try:
|
| 47 |
+
completion_kwargs = safe_deep_copy(base_kwargs)
|
| 48 |
+
# Handle dictionary fallback configurations
|
| 49 |
+
if isinstance(fallback, dict):
|
| 50 |
+
model = fallback.pop("model", original_model)
|
| 51 |
+
completion_kwargs.update(fallback)
|
| 52 |
+
else:
|
| 53 |
+
model = fallback
|
| 54 |
+
|
| 55 |
+
# Filter out internal parameters that shouldn't be sent to provider APIs
|
| 56 |
+
completion_kwargs = filter_internal_params(completion_kwargs)
|
| 57 |
+
|
| 58 |
+
response = await litellm.acompletion(
|
| 59 |
+
**completion_kwargs,
|
| 60 |
+
model=model,
|
| 61 |
+
litellm_logging_obj=litellm_logging_obj,
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
if response is not None:
|
| 65 |
+
return response
|
| 66 |
+
|
| 67 |
+
except Exception as e:
|
| 68 |
+
verbose_logger.exception(
|
| 69 |
+
f"Fallback attempt failed for model {model}: {str(e)}"
|
| 70 |
+
)
|
| 71 |
+
most_recent_exception_str = str(e)
|
| 72 |
+
continue
|
| 73 |
+
|
| 74 |
+
raise Exception(
|
| 75 |
+
f"{most_recent_exception_str}. All fallback attempts failed. Enable verbose logging with `litellm.set_verbose=True` for details."
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def completion_with_fallbacks(**kwargs):
|
| 80 |
+
return run_async_function(async_function=async_completion_with_fallbacks, **kwargs)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_blog_posts.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Pulls the latest LiteLLM blog posts from the docs RSS feed.
|
| 3 |
+
|
| 4 |
+
Falls back to the bundled local backup on any failure.
|
| 5 |
+
RSS URL is configured via litellm.blog_posts_url (or LITELLM_BLOG_POSTS_URL env var).
|
| 6 |
+
|
| 7 |
+
Disable remote fetching entirely:
|
| 8 |
+
export LITELLM_LOCAL_BLOG_POSTS=True
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
import os
|
| 13 |
+
import time
|
| 14 |
+
import xml.etree.ElementTree as ET
|
| 15 |
+
from email.utils import parsedate_to_datetime
|
| 16 |
+
from importlib.resources import files
|
| 17 |
+
from typing import Dict, List, Optional
|
| 18 |
+
|
| 19 |
+
import httpx
|
| 20 |
+
from pydantic import BaseModel
|
| 21 |
+
|
| 22 |
+
from litellm import verbose_logger
|
| 23 |
+
|
| 24 |
+
BLOG_POSTS_TTL_SECONDS: int = 3600 # 1 hour
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class BlogPost(BaseModel):
|
| 28 |
+
title: str
|
| 29 |
+
description: str
|
| 30 |
+
date: str
|
| 31 |
+
url: str
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class BlogPostsResponse(BaseModel):
|
| 35 |
+
posts: List[BlogPost]
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class GetBlogPosts:
|
| 39 |
+
"""
|
| 40 |
+
Fetches, validates, and caches LiteLLM blog posts.
|
| 41 |
+
|
| 42 |
+
- Fetches RSS feed from docs site with a 5-second timeout
|
| 43 |
+
- Parses the XML and extracts the latest blog post
|
| 44 |
+
- Caches the result in-process for BLOG_POSTS_TTL_SECONDS (1 hour)
|
| 45 |
+
- Falls back to the bundled local backup on any failure
|
| 46 |
+
"""
|
| 47 |
+
|
| 48 |
+
_cached_posts: Optional[List[Dict[str, str]]] = None
|
| 49 |
+
_last_fetch_time: float = 0.0
|
| 50 |
+
|
| 51 |
+
@staticmethod
|
| 52 |
+
def load_local_blog_posts() -> List[Dict[str, str]]:
|
| 53 |
+
"""Load the bundled local backup blog posts."""
|
| 54 |
+
content = json.loads(
|
| 55 |
+
files("litellm").joinpath("blog_posts.json").read_text(encoding="utf-8")
|
| 56 |
+
)
|
| 57 |
+
return content.get("posts", [])
|
| 58 |
+
|
| 59 |
+
@staticmethod
|
| 60 |
+
def fetch_rss_feed(url: str, timeout: int = 5) -> str:
|
| 61 |
+
"""
|
| 62 |
+
Fetch RSS XML from a remote URL.
|
| 63 |
+
|
| 64 |
+
Returns the raw XML text. Raises on network errors.
|
| 65 |
+
"""
|
| 66 |
+
response = httpx.get(url, timeout=timeout)
|
| 67 |
+
response.raise_for_status()
|
| 68 |
+
return response.text
|
| 69 |
+
|
| 70 |
+
@staticmethod
|
| 71 |
+
def parse_rss_to_posts(xml_text: str, max_posts: int = 1) -> List[Dict[str, str]]:
|
| 72 |
+
"""
|
| 73 |
+
Parse RSS XML and return a list of blog post dicts.
|
| 74 |
+
|
| 75 |
+
Extracts title, description, date (YYYY-MM-DD), and url from each <item>.
|
| 76 |
+
"""
|
| 77 |
+
root = ET.fromstring(xml_text)
|
| 78 |
+
channel = root.find("channel")
|
| 79 |
+
if channel is None:
|
| 80 |
+
raise ValueError("RSS feed missing <channel> element")
|
| 81 |
+
|
| 82 |
+
posts: List[Dict[str, str]] = []
|
| 83 |
+
for item in channel.findall("item"):
|
| 84 |
+
if len(posts) >= max_posts:
|
| 85 |
+
break
|
| 86 |
+
|
| 87 |
+
title_el = item.find("title")
|
| 88 |
+
link_el = item.find("link")
|
| 89 |
+
desc_el = item.find("description")
|
| 90 |
+
pub_date_el = item.find("pubDate")
|
| 91 |
+
|
| 92 |
+
if title_el is None or link_el is None:
|
| 93 |
+
continue
|
| 94 |
+
|
| 95 |
+
# Parse RFC 2822 date to YYYY-MM-DD
|
| 96 |
+
date_str = ""
|
| 97 |
+
if pub_date_el is not None and pub_date_el.text:
|
| 98 |
+
try:
|
| 99 |
+
dt = parsedate_to_datetime(pub_date_el.text)
|
| 100 |
+
date_str = dt.strftime("%Y-%m-%d")
|
| 101 |
+
except Exception:
|
| 102 |
+
date_str = pub_date_el.text
|
| 103 |
+
|
| 104 |
+
posts.append(
|
| 105 |
+
{
|
| 106 |
+
"title": title_el.text or "",
|
| 107 |
+
"description": desc_el.text or "" if desc_el is not None else "",
|
| 108 |
+
"date": date_str,
|
| 109 |
+
"url": link_el.text or "",
|
| 110 |
+
}
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
return posts
|
| 114 |
+
|
| 115 |
+
@staticmethod
|
| 116 |
+
def validate_blog_posts(posts: List[Dict[str, str]]) -> bool:
|
| 117 |
+
"""Return True if posts is a non-empty list."""
|
| 118 |
+
if not isinstance(posts, list) or len(posts) == 0:
|
| 119 |
+
verbose_logger.warning(
|
| 120 |
+
"LiteLLM: Parsed RSS feed has no valid posts. "
|
| 121 |
+
"Falling back to local backup.",
|
| 122 |
+
)
|
| 123 |
+
return False
|
| 124 |
+
return True
|
| 125 |
+
|
| 126 |
+
@classmethod
|
| 127 |
+
def get_blog_posts(cls, url: str) -> List[Dict[str, str]]:
|
| 128 |
+
"""
|
| 129 |
+
Return the blog posts list.
|
| 130 |
+
|
| 131 |
+
Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS.
|
| 132 |
+
Fetches from ``url`` otherwise, falling back to local backup on failure.
|
| 133 |
+
"""
|
| 134 |
+
if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
|
| 135 |
+
return cls.load_local_blog_posts()
|
| 136 |
+
|
| 137 |
+
now = time.time()
|
| 138 |
+
cached = cls._cached_posts
|
| 139 |
+
if cached is not None and (now - cls._last_fetch_time) < BLOG_POSTS_TTL_SECONDS:
|
| 140 |
+
return cached
|
| 141 |
+
|
| 142 |
+
try:
|
| 143 |
+
xml_text = cls.fetch_rss_feed(url)
|
| 144 |
+
posts = cls.parse_rss_to_posts(xml_text)
|
| 145 |
+
except Exception as e:
|
| 146 |
+
verbose_logger.warning(
|
| 147 |
+
"LiteLLM: Failed to fetch blog posts from %s: %s. "
|
| 148 |
+
"Falling back to local backup.",
|
| 149 |
+
url,
|
| 150 |
+
str(e),
|
| 151 |
+
)
|
| 152 |
+
return cls.load_local_blog_posts()
|
| 153 |
+
|
| 154 |
+
if not cls.validate_blog_posts(posts):
|
| 155 |
+
return cls.load_local_blog_posts()
|
| 156 |
+
|
| 157 |
+
cls._cached_posts = posts
|
| 158 |
+
cls._last_fetch_time = now
|
| 159 |
+
return posts
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def get_blog_posts(url: str) -> List[Dict[str, str]]:
|
| 163 |
+
"""Public entry point — returns the blog posts list."""
|
| 164 |
+
return GetBlogPosts.get_blog_posts(url=url)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_litellm_params.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional
|
| 2 |
+
|
| 3 |
+
# Pre-define optional kwargs keys as frozenset for O(1) lookups
|
| 4 |
+
# These are extracted from kwargs only if present, avoiding unnecessary .get() calls
|
| 5 |
+
_OPTIONAL_KWARGS_KEYS = frozenset(
|
| 6 |
+
{
|
| 7 |
+
"azure_ad_token",
|
| 8 |
+
"tenant_id",
|
| 9 |
+
"client_id",
|
| 10 |
+
"client_secret",
|
| 11 |
+
"azure_username",
|
| 12 |
+
"azure_password",
|
| 13 |
+
"azure_scope",
|
| 14 |
+
"timeout",
|
| 15 |
+
"bucket_name",
|
| 16 |
+
"vertex_credentials",
|
| 17 |
+
"vertex_project",
|
| 18 |
+
"vertex_location",
|
| 19 |
+
"vertex_ai_project",
|
| 20 |
+
"vertex_ai_location",
|
| 21 |
+
"vertex_ai_credentials",
|
| 22 |
+
"aws_region_name",
|
| 23 |
+
"aws_access_key_id",
|
| 24 |
+
"aws_secret_access_key",
|
| 25 |
+
"aws_session_token",
|
| 26 |
+
"aws_session_name",
|
| 27 |
+
"aws_profile_name",
|
| 28 |
+
"aws_role_name",
|
| 29 |
+
"aws_web_identity_token",
|
| 30 |
+
"aws_sts_endpoint",
|
| 31 |
+
"aws_external_id",
|
| 32 |
+
"aws_bedrock_runtime_endpoint",
|
| 33 |
+
"tpm",
|
| 34 |
+
"rpm",
|
| 35 |
+
}
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _get_base_model_from_litellm_call_metadata(
|
| 40 |
+
metadata: Optional[dict],
|
| 41 |
+
) -> Optional[str]:
|
| 42 |
+
if metadata is None:
|
| 43 |
+
return None
|
| 44 |
+
model_info = metadata.get("model_info")
|
| 45 |
+
if model_info:
|
| 46 |
+
return model_info.get("base_model")
|
| 47 |
+
return None
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def get_litellm_params(
|
| 51 |
+
api_key: Optional[str] = None,
|
| 52 |
+
force_timeout=600,
|
| 53 |
+
azure=False,
|
| 54 |
+
logger_fn=None,
|
| 55 |
+
verbose=False,
|
| 56 |
+
hugging_face=False,
|
| 57 |
+
replicate=False,
|
| 58 |
+
together_ai=False,
|
| 59 |
+
custom_llm_provider: Optional[str] = None,
|
| 60 |
+
api_base: Optional[str] = None,
|
| 61 |
+
litellm_call_id=None,
|
| 62 |
+
model_alias_map=None,
|
| 63 |
+
completion_call_id=None,
|
| 64 |
+
metadata: Optional[dict] = None,
|
| 65 |
+
model_info=None,
|
| 66 |
+
proxy_server_request=None,
|
| 67 |
+
acompletion=None,
|
| 68 |
+
aembedding=None,
|
| 69 |
+
preset_cache_key=None,
|
| 70 |
+
no_log=None,
|
| 71 |
+
input_cost_per_second=None,
|
| 72 |
+
input_cost_per_token=None,
|
| 73 |
+
output_cost_per_token=None,
|
| 74 |
+
output_cost_per_second=None,
|
| 75 |
+
cost_per_query=None,
|
| 76 |
+
cooldown_time=None,
|
| 77 |
+
text_completion=None,
|
| 78 |
+
azure_ad_token_provider=None,
|
| 79 |
+
user_continue_message=None,
|
| 80 |
+
base_model: Optional[str] = None,
|
| 81 |
+
litellm_trace_id: Optional[str] = None,
|
| 82 |
+
litellm_session_id: Optional[str] = None,
|
| 83 |
+
hf_model_name: Optional[str] = None,
|
| 84 |
+
custom_prompt_dict: Optional[dict] = None,
|
| 85 |
+
litellm_metadata: Optional[dict] = None,
|
| 86 |
+
disable_add_transform_inline_image_block: Optional[bool] = None,
|
| 87 |
+
drop_params: Optional[bool] = None,
|
| 88 |
+
prompt_id: Optional[str] = None,
|
| 89 |
+
prompt_variables: Optional[dict] = None,
|
| 90 |
+
async_call: Optional[bool] = None,
|
| 91 |
+
ssl_verify: Optional[bool] = None,
|
| 92 |
+
merge_reasoning_content_in_choices: Optional[bool] = None,
|
| 93 |
+
use_litellm_proxy: Optional[bool] = None,
|
| 94 |
+
api_version: Optional[str] = None,
|
| 95 |
+
max_retries: Optional[int] = None,
|
| 96 |
+
litellm_request_debug: Optional[bool] = None,
|
| 97 |
+
**kwargs,
|
| 98 |
+
) -> dict:
|
| 99 |
+
# Derive litellm_session_id / litellm_trace_id from metadata when not provided (call chaining)
|
| 100 |
+
_meta = metadata or {}
|
| 101 |
+
if litellm_session_id is None:
|
| 102 |
+
litellm_session_id = _meta.get("session_id") or _meta.get("trace_id")
|
| 103 |
+
if litellm_trace_id is None:
|
| 104 |
+
litellm_trace_id = _meta.get("trace_id") or _meta.get("session_id")
|
| 105 |
+
|
| 106 |
+
# Build base dict with explicit parameters (always included)
|
| 107 |
+
litellm_params = {
|
| 108 |
+
"acompletion": acompletion,
|
| 109 |
+
"api_key": api_key,
|
| 110 |
+
"force_timeout": force_timeout,
|
| 111 |
+
"logger_fn": logger_fn,
|
| 112 |
+
"verbose": verbose,
|
| 113 |
+
"custom_llm_provider": custom_llm_provider,
|
| 114 |
+
"api_base": api_base,
|
| 115 |
+
"litellm_call_id": litellm_call_id,
|
| 116 |
+
"model_alias_map": model_alias_map,
|
| 117 |
+
"completion_call_id": completion_call_id,
|
| 118 |
+
"aembedding": aembedding,
|
| 119 |
+
"metadata": metadata,
|
| 120 |
+
"model_info": model_info,
|
| 121 |
+
"proxy_server_request": proxy_server_request,
|
| 122 |
+
"preset_cache_key": preset_cache_key,
|
| 123 |
+
"no-log": no_log or kwargs.get("no-log"),
|
| 124 |
+
"stream_response": {}, # litellm_call_id: ModelResponse Dict
|
| 125 |
+
"input_cost_per_token": input_cost_per_token,
|
| 126 |
+
"input_cost_per_second": input_cost_per_second,
|
| 127 |
+
"output_cost_per_token": output_cost_per_token,
|
| 128 |
+
"output_cost_per_second": output_cost_per_second,
|
| 129 |
+
"cost_per_query": cost_per_query,
|
| 130 |
+
"cooldown_time": cooldown_time,
|
| 131 |
+
"text_completion": text_completion,
|
| 132 |
+
"azure_ad_token_provider": azure_ad_token_provider,
|
| 133 |
+
"user_continue_message": user_continue_message,
|
| 134 |
+
"base_model": base_model
|
| 135 |
+
or (
|
| 136 |
+
_get_base_model_from_litellm_call_metadata(metadata=metadata)
|
| 137 |
+
if metadata
|
| 138 |
+
else None
|
| 139 |
+
),
|
| 140 |
+
"litellm_trace_id": litellm_trace_id,
|
| 141 |
+
"litellm_session_id": litellm_session_id,
|
| 142 |
+
"hf_model_name": hf_model_name,
|
| 143 |
+
"custom_prompt_dict": custom_prompt_dict,
|
| 144 |
+
"litellm_metadata": litellm_metadata,
|
| 145 |
+
"disable_add_transform_inline_image_block": disable_add_transform_inline_image_block,
|
| 146 |
+
"drop_params": drop_params,
|
| 147 |
+
"prompt_id": prompt_id,
|
| 148 |
+
"prompt_variables": prompt_variables,
|
| 149 |
+
"async_call": async_call,
|
| 150 |
+
"ssl_verify": ssl_verify,
|
| 151 |
+
"merge_reasoning_content_in_choices": merge_reasoning_content_in_choices,
|
| 152 |
+
"api_version": api_version,
|
| 153 |
+
"max_retries": max_retries,
|
| 154 |
+
"use_litellm_proxy": use_litellm_proxy,
|
| 155 |
+
"litellm_request_debug": litellm_request_debug,
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
# Sparse extraction: only add kwargs keys that are actually present
|
| 159 |
+
if kwargs:
|
| 160 |
+
for key in _OPTIONAL_KWARGS_KEYS:
|
| 161 |
+
if key in kwargs:
|
| 162 |
+
litellm_params[key] = kwargs[key]
|
| 163 |
+
|
| 164 |
+
return litellm_params
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py
ADDED
|
@@ -0,0 +1,958 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional, Tuple
|
| 2 |
+
|
| 3 |
+
import litellm
|
| 4 |
+
from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
|
| 5 |
+
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
| 6 |
+
from litellm.secret_managers.main import get_secret, get_secret_str
|
| 7 |
+
|
| 8 |
+
from ..types.router import LiteLLM_Params
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _is_non_openai_azure_model(model: str) -> bool:
|
| 12 |
+
try:
|
| 13 |
+
model_name = model.split("/", 1)[1]
|
| 14 |
+
if (
|
| 15 |
+
model_name in litellm.cohere_chat_models
|
| 16 |
+
or f"mistral/{model_name}" in litellm.mistral_chat_models
|
| 17 |
+
):
|
| 18 |
+
return True
|
| 19 |
+
except Exception:
|
| 20 |
+
return False
|
| 21 |
+
return False
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _is_azure_claude_model(model: str) -> bool:
|
| 25 |
+
"""
|
| 26 |
+
Check if a model name contains 'claude' (case-insensitive).
|
| 27 |
+
Used to detect Claude models that need Anthropic-specific handling.
|
| 28 |
+
"""
|
| 29 |
+
try:
|
| 30 |
+
model_lower = model.lower()
|
| 31 |
+
return "claude" in model_lower or model_lower.startswith("claude")
|
| 32 |
+
except Exception:
|
| 33 |
+
return False
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def handle_cohere_chat_model_custom_llm_provider(
|
| 37 |
+
model: str, custom_llm_provider: Optional[str] = None
|
| 38 |
+
) -> Tuple[str, Optional[str]]:
|
| 39 |
+
"""
|
| 40 |
+
if user sets model = "cohere/command-r" -> use custom_llm_provider = "cohere_chat"
|
| 41 |
+
|
| 42 |
+
Args:
|
| 43 |
+
model:
|
| 44 |
+
custom_llm_provider:
|
| 45 |
+
|
| 46 |
+
Returns:
|
| 47 |
+
model, custom_llm_provider
|
| 48 |
+
"""
|
| 49 |
+
|
| 50 |
+
if custom_llm_provider:
|
| 51 |
+
if custom_llm_provider == "cohere" and model in litellm.cohere_chat_models:
|
| 52 |
+
return model, "cohere_chat"
|
| 53 |
+
|
| 54 |
+
if model and "/" in model:
|
| 55 |
+
_custom_llm_provider, _model = model.split("/", 1)
|
| 56 |
+
if (
|
| 57 |
+
_custom_llm_provider
|
| 58 |
+
and _custom_llm_provider == "cohere"
|
| 59 |
+
and _model in litellm.cohere_chat_models
|
| 60 |
+
):
|
| 61 |
+
return _model, "cohere_chat"
|
| 62 |
+
|
| 63 |
+
return model, custom_llm_provider
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def handle_anthropic_text_model_custom_llm_provider(
|
| 67 |
+
model: str, custom_llm_provider: Optional[str] = None
|
| 68 |
+
) -> Tuple[str, Optional[str]]:
|
| 69 |
+
"""
|
| 70 |
+
if user sets model = "anthropic/claude-2" -> use custom_llm_provider = "anthropic_text"
|
| 71 |
+
|
| 72 |
+
Args:
|
| 73 |
+
model:
|
| 74 |
+
custom_llm_provider:
|
| 75 |
+
|
| 76 |
+
Returns:
|
| 77 |
+
model, custom_llm_provider
|
| 78 |
+
"""
|
| 79 |
+
|
| 80 |
+
if custom_llm_provider:
|
| 81 |
+
if (
|
| 82 |
+
custom_llm_provider == "anthropic"
|
| 83 |
+
and litellm.AnthropicTextConfig._is_anthropic_text_model(model)
|
| 84 |
+
):
|
| 85 |
+
return model, "anthropic_text"
|
| 86 |
+
|
| 87 |
+
if model and "/" in model:
|
| 88 |
+
_custom_llm_provider, _model = model.split("/", 1)
|
| 89 |
+
if (
|
| 90 |
+
_custom_llm_provider
|
| 91 |
+
and _custom_llm_provider == "anthropic"
|
| 92 |
+
and litellm.AnthropicTextConfig._is_anthropic_text_model(_model)
|
| 93 |
+
):
|
| 94 |
+
return _model, "anthropic_text"
|
| 95 |
+
|
| 96 |
+
return model, custom_llm_provider
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def get_llm_provider( # noqa: PLR0915
|
| 100 |
+
model: str,
|
| 101 |
+
custom_llm_provider: Optional[str] = None,
|
| 102 |
+
api_base: Optional[str] = None,
|
| 103 |
+
api_key: Optional[str] = None,
|
| 104 |
+
litellm_params: Optional[LiteLLM_Params] = None,
|
| 105 |
+
) -> Tuple[str, str, Optional[str], Optional[str]]:
|
| 106 |
+
"""
|
| 107 |
+
Returns the provider for a given model name - e.g. 'azure/chatgpt-v-2' -> 'azure'
|
| 108 |
+
|
| 109 |
+
For router -> Can also give the whole litellm param dict -> this function will extract the relevant details
|
| 110 |
+
|
| 111 |
+
Raises Error - if unable to map model to a provider
|
| 112 |
+
|
| 113 |
+
Return model, custom_llm_provider, dynamic_api_key, api_base
|
| 114 |
+
"""
|
| 115 |
+
try:
|
| 116 |
+
# Early validation - model is required
|
| 117 |
+
if model is None:
|
| 118 |
+
raise ValueError(
|
| 119 |
+
"model parameter is required but was None. Please provide a valid model name."
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
if litellm.LiteLLMProxyChatConfig._should_use_litellm_proxy_by_default(
|
| 123 |
+
litellm_params=litellm_params
|
| 124 |
+
):
|
| 125 |
+
return litellm.LiteLLMProxyChatConfig.litellm_proxy_get_custom_llm_provider_info(
|
| 126 |
+
model=model, api_base=api_base, api_key=api_key
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
+
## IF LITELLM PARAMS GIVEN ##
|
| 130 |
+
if litellm_params:
|
| 131 |
+
assert (
|
| 132 |
+
custom_llm_provider is None and api_base is None and api_key is None
|
| 133 |
+
), "Either pass in litellm_params or the custom_llm_provider/api_base/api_key. Otherwise, these values will be overriden."
|
| 134 |
+
custom_llm_provider = litellm_params.custom_llm_provider
|
| 135 |
+
api_base = litellm_params.api_base
|
| 136 |
+
api_key = litellm_params.api_key
|
| 137 |
+
|
| 138 |
+
dynamic_api_key = None
|
| 139 |
+
# check if llm provider provided
|
| 140 |
+
# AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere
|
| 141 |
+
# If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus
|
| 142 |
+
if model.split("/", 1)[0] == "azure":
|
| 143 |
+
if _is_non_openai_azure_model(model):
|
| 144 |
+
custom_llm_provider = "openai"
|
| 145 |
+
return model, custom_llm_provider, dynamic_api_key, api_base
|
| 146 |
+
|
| 147 |
+
### Handle cases when custom_llm_provider is set to cohere/command-r-plus but it should use cohere_chat route
|
| 148 |
+
model, custom_llm_provider = handle_cohere_chat_model_custom_llm_provider(
|
| 149 |
+
model, custom_llm_provider
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
model, custom_llm_provider = handle_anthropic_text_model_custom_llm_provider(
|
| 153 |
+
model, custom_llm_provider
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
if custom_llm_provider and (
|
| 157 |
+
model.split("/")[0] != custom_llm_provider
|
| 158 |
+
): # handle scenario where model="azure/*" and custom_llm_provider="azure"
|
| 159 |
+
model = custom_llm_provider + "/" + model
|
| 160 |
+
|
| 161 |
+
# Native OpenRouter models have IDs like "openrouter/free" where the
|
| 162 |
+
# "openrouter/" prefix is part of the actual model name on the API.
|
| 163 |
+
# When called from a bridge (e.g. anthropic_messages adapter),
|
| 164 |
+
# custom_llm_provider is already resolved, so return early to prevent
|
| 165 |
+
# the provider-list stripping below from removing the prefix.
|
| 166 |
+
if custom_llm_provider == "openrouter" and model.startswith("openrouter/"):
|
| 167 |
+
return model, custom_llm_provider, dynamic_api_key, api_base
|
| 168 |
+
|
| 169 |
+
if api_key and api_key.startswith("os.environ/"):
|
| 170 |
+
dynamic_api_key = get_secret_str(api_key)
|
| 171 |
+
|
| 172 |
+
# Check JSON-configured providers FIRST (before enum-based provider_list)
|
| 173 |
+
provider_prefix = model.split("/", 1)[0]
|
| 174 |
+
if len(model.split("/")) > 1 and JSONProviderRegistry.exists(provider_prefix):
|
| 175 |
+
return _get_openai_compatible_provider_info(
|
| 176 |
+
model=model,
|
| 177 |
+
api_base=api_base,
|
| 178 |
+
api_key=api_key,
|
| 179 |
+
dynamic_api_key=dynamic_api_key,
|
| 180 |
+
)
|
| 181 |
+
|
| 182 |
+
# check if llm provider part of model name
|
| 183 |
+
|
| 184 |
+
if (
|
| 185 |
+
model.split("/", 1)[0] in litellm.provider_list
|
| 186 |
+
and model.split("/", 1)[0] not in litellm.model_list_set
|
| 187 |
+
and len(model.split("/"))
|
| 188 |
+
> 1 # handle edge case where user passes in `litellm --model mistral` https://github.com/BerriAI/litellm/issues/1351
|
| 189 |
+
):
|
| 190 |
+
return _get_openai_compatible_provider_info(
|
| 191 |
+
model=model,
|
| 192 |
+
api_base=api_base,
|
| 193 |
+
api_key=api_key,
|
| 194 |
+
dynamic_api_key=dynamic_api_key,
|
| 195 |
+
)
|
| 196 |
+
elif model.split("/", 1)[0] in litellm.provider_list:
|
| 197 |
+
custom_llm_provider = model.split("/", 1)[0]
|
| 198 |
+
model = model.split("/", 1)[1]
|
| 199 |
+
if api_base is not None and not isinstance(api_base, str):
|
| 200 |
+
raise Exception(
|
| 201 |
+
"api base needs to be a string. api_base={}".format(api_base)
|
| 202 |
+
)
|
| 203 |
+
if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
|
| 204 |
+
raise Exception(
|
| 205 |
+
"dynamic_api_key needs to be a string. dynamic_api_key={}".format(
|
| 206 |
+
dynamic_api_key
|
| 207 |
+
)
|
| 208 |
+
)
|
| 209 |
+
return model, custom_llm_provider, dynamic_api_key, api_base
|
| 210 |
+
# check if api base is a known openai compatible endpoint
|
| 211 |
+
if api_base:
|
| 212 |
+
for endpoint in litellm.openai_compatible_endpoints:
|
| 213 |
+
if endpoint in api_base:
|
| 214 |
+
if endpoint == "api.perplexity.ai":
|
| 215 |
+
custom_llm_provider = "perplexity"
|
| 216 |
+
dynamic_api_key = get_secret_str("PERPLEXITYAI_API_KEY")
|
| 217 |
+
elif endpoint == "api.endpoints.anyscale.com/v1":
|
| 218 |
+
custom_llm_provider = "anyscale"
|
| 219 |
+
dynamic_api_key = get_secret_str("ANYSCALE_API_KEY")
|
| 220 |
+
elif endpoint == "api.deepinfra.com/v1/openai":
|
| 221 |
+
custom_llm_provider = "deepinfra"
|
| 222 |
+
dynamic_api_key = get_secret_str("DEEPINFRA_API_KEY")
|
| 223 |
+
elif endpoint == "api.mistral.ai/v1":
|
| 224 |
+
custom_llm_provider = "mistral"
|
| 225 |
+
dynamic_api_key = get_secret_str("MISTRAL_API_KEY")
|
| 226 |
+
elif endpoint == "api.groq.com/openai/v1":
|
| 227 |
+
custom_llm_provider = "groq"
|
| 228 |
+
dynamic_api_key = get_secret_str("GROQ_API_KEY")
|
| 229 |
+
elif endpoint == "https://integrate.api.nvidia.com/v1":
|
| 230 |
+
custom_llm_provider = "nvidia_nim"
|
| 231 |
+
dynamic_api_key = get_secret_str("NVIDIA_NIM_API_KEY")
|
| 232 |
+
elif endpoint == "https://api.cerebras.ai/v1":
|
| 233 |
+
custom_llm_provider = "cerebras"
|
| 234 |
+
dynamic_api_key = get_secret_str("CEREBRAS_API_KEY")
|
| 235 |
+
elif endpoint == "https://inference.baseten.co/v1":
|
| 236 |
+
custom_llm_provider = "baseten"
|
| 237 |
+
dynamic_api_key = get_secret_str("BASETEN_API_KEY")
|
| 238 |
+
elif endpoint == "https://api.sambanova.ai/v1":
|
| 239 |
+
custom_llm_provider = "sambanova"
|
| 240 |
+
dynamic_api_key = get_secret_str("SAMBANOVA_API_KEY")
|
| 241 |
+
elif endpoint == "https://api.ai21.com/studio/v1":
|
| 242 |
+
custom_llm_provider = "ai21_chat"
|
| 243 |
+
dynamic_api_key = get_secret_str("AI21_API_KEY")
|
| 244 |
+
elif endpoint == "codestral.mistral.ai/v1/chat/completions":
|
| 245 |
+
custom_llm_provider = "codestral"
|
| 246 |
+
dynamic_api_key = get_secret_str("CODESTRAL_API_KEY")
|
| 247 |
+
elif endpoint == "codestral.mistral.ai/v1/fim/completions":
|
| 248 |
+
custom_llm_provider = "text-completion-codestral"
|
| 249 |
+
dynamic_api_key = get_secret_str("CODESTRAL_API_KEY")
|
| 250 |
+
elif endpoint == "app.empower.dev/api/v1":
|
| 251 |
+
custom_llm_provider = "empower"
|
| 252 |
+
dynamic_api_key = get_secret_str("EMPOWER_API_KEY")
|
| 253 |
+
elif endpoint == "api.deepseek.com/v1":
|
| 254 |
+
custom_llm_provider = "deepseek"
|
| 255 |
+
dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY")
|
| 256 |
+
elif endpoint == "ollama.com":
|
| 257 |
+
custom_llm_provider = "ollama"
|
| 258 |
+
dynamic_api_key = get_secret_str("OLLAMA_API_KEY")
|
| 259 |
+
elif endpoint == "https://api.friendli.ai/serverless/v1":
|
| 260 |
+
custom_llm_provider = "friendliai"
|
| 261 |
+
dynamic_api_key = get_secret_str(
|
| 262 |
+
"FRIENDLIAI_API_KEY"
|
| 263 |
+
) or get_secret("FRIENDLI_TOKEN")
|
| 264 |
+
elif endpoint == "api.galadriel.com/v1":
|
| 265 |
+
custom_llm_provider = "galadriel"
|
| 266 |
+
dynamic_api_key = get_secret_str("GALADRIEL_API_KEY")
|
| 267 |
+
elif endpoint == "https://api.llama.com/compat/v1":
|
| 268 |
+
custom_llm_provider = "meta_llama"
|
| 269 |
+
dynamic_api_key = api_key or get_secret_str("LLAMA_API_KEY")
|
| 270 |
+
elif endpoint == "https://api.featherless.ai/v1":
|
| 271 |
+
custom_llm_provider = "featherless_ai"
|
| 272 |
+
dynamic_api_key = get_secret_str("FEATHERLESS_AI_API_KEY")
|
| 273 |
+
elif endpoint == litellm.NscaleConfig.API_BASE_URL:
|
| 274 |
+
custom_llm_provider = "nscale"
|
| 275 |
+
dynamic_api_key = litellm.NscaleConfig.get_api_key()
|
| 276 |
+
elif endpoint == "dashscope-intl.aliyuncs.com/compatible-mode/v1":
|
| 277 |
+
custom_llm_provider = "dashscope"
|
| 278 |
+
dynamic_api_key = get_secret_str("DASHSCOPE_API_KEY")
|
| 279 |
+
elif endpoint == "api.moonshot.ai/v1":
|
| 280 |
+
custom_llm_provider = "moonshot"
|
| 281 |
+
dynamic_api_key = get_secret_str("MOONSHOT_API_KEY")
|
| 282 |
+
elif (
|
| 283 |
+
endpoint == "api.minimax.io/anthropic"
|
| 284 |
+
or endpoint == "api.minimaxi.com/anthropic"
|
| 285 |
+
):
|
| 286 |
+
custom_llm_provider = "minimax"
|
| 287 |
+
dynamic_api_key = get_secret_str("MINIMAX_API_KEY")
|
| 288 |
+
elif (
|
| 289 |
+
endpoint == "api.minimax.io/v1"
|
| 290 |
+
or endpoint == "api.minimaxi.com/v1"
|
| 291 |
+
):
|
| 292 |
+
custom_llm_provider = "minimax"
|
| 293 |
+
dynamic_api_key = get_secret_str("MINIMAX_API_KEY")
|
| 294 |
+
elif endpoint == "platform.publicai.co/v1":
|
| 295 |
+
custom_llm_provider = "publicai"
|
| 296 |
+
dynamic_api_key = get_secret_str("PUBLICAI_API_KEY")
|
| 297 |
+
elif endpoint == "https://api.synthetic.new/openai/v1":
|
| 298 |
+
custom_llm_provider = "synthetic"
|
| 299 |
+
dynamic_api_key = get_secret_str("SYNTHETIC_API_KEY")
|
| 300 |
+
elif endpoint == "https://api.stima.tech/v1":
|
| 301 |
+
custom_llm_provider = "apertis"
|
| 302 |
+
dynamic_api_key = get_secret_str("STIMA_API_KEY")
|
| 303 |
+
elif endpoint == "https://nano-gpt.com/api/v1":
|
| 304 |
+
custom_llm_provider = "nano-gpt"
|
| 305 |
+
dynamic_api_key = get_secret_str("NANOGPT_API_KEY")
|
| 306 |
+
elif endpoint == "https://api.poe.com/v1":
|
| 307 |
+
custom_llm_provider = "poe"
|
| 308 |
+
dynamic_api_key = get_secret_str("POE_API_KEY")
|
| 309 |
+
elif endpoint == "https://llm.chutes.ai/v1/":
|
| 310 |
+
custom_llm_provider = "chutes"
|
| 311 |
+
dynamic_api_key = get_secret_str("CHUTES_API_KEY")
|
| 312 |
+
elif endpoint == "https://api.v0.dev/v1":
|
| 313 |
+
custom_llm_provider = "v0"
|
| 314 |
+
dynamic_api_key = get_secret_str("V0_API_KEY")
|
| 315 |
+
elif endpoint == "https://api.lambda.ai/v1":
|
| 316 |
+
custom_llm_provider = "lambda_ai"
|
| 317 |
+
dynamic_api_key = get_secret_str("LAMBDA_API_KEY")
|
| 318 |
+
elif endpoint == "https://api.hyperbolic.xyz/v1":
|
| 319 |
+
custom_llm_provider = "hyperbolic"
|
| 320 |
+
dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY")
|
| 321 |
+
elif endpoint == "https://ai-gateway.vercel.sh/v1":
|
| 322 |
+
custom_llm_provider = "vercel_ai_gateway"
|
| 323 |
+
dynamic_api_key = get_secret_str("VERCEL_AI_GATEWAY_API_KEY")
|
| 324 |
+
elif endpoint == "https://api.inference.wandb.ai/v1":
|
| 325 |
+
custom_llm_provider = "wandb"
|
| 326 |
+
dynamic_api_key = get_secret_str("WANDB_API_KEY")
|
| 327 |
+
|
| 328 |
+
if api_base is not None and not isinstance(api_base, str):
|
| 329 |
+
raise Exception(
|
| 330 |
+
"api base needs to be a string. api_base={}".format(
|
| 331 |
+
api_base
|
| 332 |
+
)
|
| 333 |
+
)
|
| 334 |
+
if dynamic_api_key is not None and not isinstance(
|
| 335 |
+
dynamic_api_key, str
|
| 336 |
+
):
|
| 337 |
+
raise Exception(
|
| 338 |
+
"dynamic_api_key needs to be a string. dynamic_api_key={}".format(
|
| 339 |
+
dynamic_api_key
|
| 340 |
+
)
|
| 341 |
+
)
|
| 342 |
+
return model, custom_llm_provider, dynamic_api_key, api_base # type: ignore
|
| 343 |
+
|
| 344 |
+
# check if model in known model provider list -> for huggingface models, raise exception as they don't have a fixed provider (can be togetherai, anyscale, baseten, runpod, et.)
|
| 345 |
+
## openai - chatcompletion + text completion
|
| 346 |
+
if (
|
| 347 |
+
model in litellm.open_ai_chat_completion_models
|
| 348 |
+
or "ft:gpt-3.5-turbo" in model
|
| 349 |
+
or "ft:gpt-4" in model # catches ft:gpt-4-0613, ft:gpt-4o
|
| 350 |
+
or model in litellm.openai_image_generation_models
|
| 351 |
+
or model in litellm.openai_video_generation_models
|
| 352 |
+
):
|
| 353 |
+
custom_llm_provider = "openai"
|
| 354 |
+
elif model in litellm.open_ai_text_completion_models:
|
| 355 |
+
custom_llm_provider = "text-completion-openai"
|
| 356 |
+
## anthropic
|
| 357 |
+
elif model in litellm.anthropic_models:
|
| 358 |
+
if litellm.AnthropicTextConfig._is_anthropic_text_model(model):
|
| 359 |
+
custom_llm_provider = "anthropic_text"
|
| 360 |
+
else:
|
| 361 |
+
custom_llm_provider = "anthropic"
|
| 362 |
+
## cohere
|
| 363 |
+
elif model in litellm.cohere_models or model in litellm.cohere_embedding_models:
|
| 364 |
+
custom_llm_provider = "cohere"
|
| 365 |
+
## cohere chat models
|
| 366 |
+
elif model in litellm.cohere_chat_models:
|
| 367 |
+
custom_llm_provider = "cohere_chat"
|
| 368 |
+
## replicate
|
| 369 |
+
elif model in litellm.replicate_models or (
|
| 370 |
+
":" in model and len(model) > REPLICATE_MODEL_NAME_WITH_ID_LENGTH
|
| 371 |
+
):
|
| 372 |
+
model_parts = model.split(":")
|
| 373 |
+
if (
|
| 374 |
+
len(model_parts) > 1
|
| 375 |
+
and len(model_parts[1]) == REPLICATE_MODEL_NAME_WITH_ID_LENGTH
|
| 376 |
+
): ## checks if model name has a 64 digit code - e.g. "meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3"
|
| 377 |
+
custom_llm_provider = "replicate"
|
| 378 |
+
elif model in litellm.replicate_models:
|
| 379 |
+
custom_llm_provider = "replicate"
|
| 380 |
+
## openrouter
|
| 381 |
+
elif model in litellm.openrouter_models:
|
| 382 |
+
custom_llm_provider = "openrouter"
|
| 383 |
+
## maritalk
|
| 384 |
+
elif model in litellm.maritalk_models:
|
| 385 |
+
custom_llm_provider = "maritalk"
|
| 386 |
+
## vertex - text + chat + language (gemini) models
|
| 387 |
+
elif (
|
| 388 |
+
model in litellm.vertex_chat_models
|
| 389 |
+
or model in litellm.vertex_code_chat_models
|
| 390 |
+
or model in litellm.vertex_text_models
|
| 391 |
+
or model in litellm.vertex_code_text_models
|
| 392 |
+
or model in litellm.vertex_language_models
|
| 393 |
+
or model in litellm.vertex_embedding_models
|
| 394 |
+
or model in litellm.vertex_vision_models
|
| 395 |
+
or model in litellm.vertex_ai_image_models
|
| 396 |
+
or model in litellm.vertex_ai_video_models
|
| 397 |
+
):
|
| 398 |
+
custom_llm_provider = "vertex_ai"
|
| 399 |
+
## ai21
|
| 400 |
+
elif model in litellm.ai21_chat_models or model in litellm.ai21_models:
|
| 401 |
+
custom_llm_provider = "ai21_chat"
|
| 402 |
+
api_base = (
|
| 403 |
+
api_base
|
| 404 |
+
or get_secret("AI21_API_BASE")
|
| 405 |
+
or "https://api.ai21.com/studio/v1"
|
| 406 |
+
) # type: ignore
|
| 407 |
+
dynamic_api_key = api_key or get_secret("AI21_API_KEY")
|
| 408 |
+
## aleph_alpha
|
| 409 |
+
elif model in litellm.aleph_alpha_models:
|
| 410 |
+
custom_llm_provider = "aleph_alpha"
|
| 411 |
+
## baseten
|
| 412 |
+
elif model in litellm.baseten_models:
|
| 413 |
+
custom_llm_provider = "baseten"
|
| 414 |
+
## nlp_cloud
|
| 415 |
+
elif model in litellm.nlp_cloud_models:
|
| 416 |
+
custom_llm_provider = "nlp_cloud"
|
| 417 |
+
## petals
|
| 418 |
+
elif model in litellm.petals_models:
|
| 419 |
+
custom_llm_provider = "petals"
|
| 420 |
+
## bedrock
|
| 421 |
+
elif (
|
| 422 |
+
model in litellm.bedrock_models
|
| 423 |
+
or model in litellm.bedrock_embedding_models
|
| 424 |
+
or model in litellm.bedrock_converse_models
|
| 425 |
+
):
|
| 426 |
+
custom_llm_provider = "bedrock"
|
| 427 |
+
elif model in litellm.watsonx_models:
|
| 428 |
+
custom_llm_provider = "watsonx"
|
| 429 |
+
# openai embeddings
|
| 430 |
+
elif model in litellm.open_ai_embedding_models:
|
| 431 |
+
custom_llm_provider = "openai"
|
| 432 |
+
elif model in litellm.empower_models:
|
| 433 |
+
custom_llm_provider = "empower"
|
| 434 |
+
elif model in litellm.gradient_ai_models:
|
| 435 |
+
custom_llm_provider = "gradient_ai"
|
| 436 |
+
elif model == "*":
|
| 437 |
+
custom_llm_provider = "openai"
|
| 438 |
+
# bytez models
|
| 439 |
+
elif model.startswith("bytez/"):
|
| 440 |
+
custom_llm_provider = "bytez"
|
| 441 |
+
elif model.startswith("lemonade/"):
|
| 442 |
+
custom_llm_provider = "lemonade"
|
| 443 |
+
elif model.startswith("heroku/"):
|
| 444 |
+
custom_llm_provider = "heroku"
|
| 445 |
+
# cometapi models
|
| 446 |
+
elif model.startswith("cometapi/"):
|
| 447 |
+
custom_llm_provider = "cometapi"
|
| 448 |
+
elif model.startswith("oci/"):
|
| 449 |
+
custom_llm_provider = "oci"
|
| 450 |
+
elif model.startswith("compactifai/"):
|
| 451 |
+
custom_llm_provider = "compactifai"
|
| 452 |
+
elif model.startswith("ovhcloud/"):
|
| 453 |
+
custom_llm_provider = "ovhcloud"
|
| 454 |
+
elif model.startswith("lemonade/"):
|
| 455 |
+
custom_llm_provider = "lemonade"
|
| 456 |
+
elif model.startswith("clarifai/"):
|
| 457 |
+
custom_llm_provider = "clarifai"
|
| 458 |
+
elif model.startswith("amazon_nova"):
|
| 459 |
+
custom_llm_provider = "amazon_nova"
|
| 460 |
+
elif model.startswith("sap/"):
|
| 461 |
+
custom_llm_provider = "sap"
|
| 462 |
+
if not custom_llm_provider:
|
| 463 |
+
if litellm.suppress_debug_info is False:
|
| 464 |
+
print() # noqa
|
| 465 |
+
print( # noqa
|
| 466 |
+
"\033[1;31mProvider List: https://docs.litellm.ai/docs/providers\033[0m" # noqa
|
| 467 |
+
) # noqa
|
| 468 |
+
print() # noqa
|
| 469 |
+
error_str = f"LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model={model}\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers"
|
| 470 |
+
# maps to openai.NotFoundError, this is raised when openai does not recognize the llm
|
| 471 |
+
raise litellm.exceptions.BadRequestError( # type: ignore
|
| 472 |
+
message=error_str,
|
| 473 |
+
model=model,
|
| 474 |
+
response=None,
|
| 475 |
+
llm_provider="",
|
| 476 |
+
)
|
| 477 |
+
if api_base is not None and not isinstance(api_base, str):
|
| 478 |
+
raise Exception(
|
| 479 |
+
"api base needs to be a string. api_base={}".format(api_base)
|
| 480 |
+
)
|
| 481 |
+
if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
|
| 482 |
+
raise Exception(
|
| 483 |
+
"dynamic_api_key needs to be a string. dynamic_api_key={}".format(
|
| 484 |
+
dynamic_api_key
|
| 485 |
+
)
|
| 486 |
+
)
|
| 487 |
+
return model, custom_llm_provider, dynamic_api_key, api_base
|
| 488 |
+
except Exception as e:
|
| 489 |
+
if isinstance(e, litellm.exceptions.BadRequestError):
|
| 490 |
+
raise e
|
| 491 |
+
else:
|
| 492 |
+
error_str = (
|
| 493 |
+
f"GetLLMProvider Exception - {str(e)}\n\noriginal model: {model}"
|
| 494 |
+
)
|
| 495 |
+
raise litellm.exceptions.BadRequestError( # type: ignore
|
| 496 |
+
message=f"GetLLMProvider Exception - {str(e)}\n\noriginal model: {model}",
|
| 497 |
+
model=model,
|
| 498 |
+
response=None,
|
| 499 |
+
llm_provider="",
|
| 500 |
+
)
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
def _get_openai_compatible_provider_info( # noqa: PLR0915
|
| 504 |
+
model: str,
|
| 505 |
+
api_base: Optional[str],
|
| 506 |
+
api_key: Optional[str],
|
| 507 |
+
dynamic_api_key: Optional[str],
|
| 508 |
+
) -> Tuple[str, str, Optional[str], Optional[str]]:
|
| 509 |
+
"""
|
| 510 |
+
Returns:
|
| 511 |
+
Tuple[str, str, Optional[str], Optional[str]]:
|
| 512 |
+
model: str
|
| 513 |
+
custom_llm_provider: str
|
| 514 |
+
dynamic_api_key: Optional[str]
|
| 515 |
+
api_base: Optional[str]
|
| 516 |
+
"""
|
| 517 |
+
|
| 518 |
+
custom_llm_provider = model.split("/", 1)[0]
|
| 519 |
+
model = model.split("/", 1)[1]
|
| 520 |
+
|
| 521 |
+
# Check JSON providers FIRST (before hardcoded ones)
|
| 522 |
+
from litellm.llms.openai_like.dynamic_config import create_config_class
|
| 523 |
+
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
| 524 |
+
|
| 525 |
+
if JSONProviderRegistry.exists(custom_llm_provider):
|
| 526 |
+
provider_config = JSONProviderRegistry.get(custom_llm_provider)
|
| 527 |
+
if provider_config is None:
|
| 528 |
+
raise ValueError(f"Provider {custom_llm_provider} not found")
|
| 529 |
+
config_class = create_config_class(provider_config)
|
| 530 |
+
api_base, dynamic_api_key = config_class()._get_openai_compatible_provider_info(
|
| 531 |
+
api_base, api_key
|
| 532 |
+
)
|
| 533 |
+
return model, custom_llm_provider, dynamic_api_key, api_base
|
| 534 |
+
|
| 535 |
+
if custom_llm_provider == "perplexity":
|
| 536 |
+
# perplexity is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.perplexity.ai
|
| 537 |
+
(
|
| 538 |
+
api_base,
|
| 539 |
+
dynamic_api_key,
|
| 540 |
+
) = litellm.PerplexityChatConfig()._get_openai_compatible_provider_info(
|
| 541 |
+
api_base, api_key
|
| 542 |
+
)
|
| 543 |
+
elif custom_llm_provider == "aiohttp_openai":
|
| 544 |
+
return model, "aiohttp_openai", api_key, api_base
|
| 545 |
+
elif custom_llm_provider == "anyscale":
|
| 546 |
+
# anyscale is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
|
| 547 |
+
api_base = api_base or get_secret_str("ANYSCALE_API_BASE") or "https://api.endpoints.anyscale.com/v1" # type: ignore
|
| 548 |
+
dynamic_api_key = api_key or get_secret_str("ANYSCALE_API_KEY")
|
| 549 |
+
elif custom_llm_provider == "deepinfra":
|
| 550 |
+
(
|
| 551 |
+
api_base,
|
| 552 |
+
dynamic_api_key,
|
| 553 |
+
) = litellm.DeepInfraConfig()._get_openai_compatible_provider_info(
|
| 554 |
+
api_base, api_key
|
| 555 |
+
)
|
| 556 |
+
elif custom_llm_provider == "empower":
|
| 557 |
+
api_base = (
|
| 558 |
+
api_base
|
| 559 |
+
or get_secret("EMPOWER_API_BASE")
|
| 560 |
+
or "https://app.empower.dev/api/v1"
|
| 561 |
+
) # type: ignore
|
| 562 |
+
dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY")
|
| 563 |
+
elif custom_llm_provider == "groq":
|
| 564 |
+
(
|
| 565 |
+
api_base,
|
| 566 |
+
dynamic_api_key,
|
| 567 |
+
) = litellm.GroqChatConfig()._get_openai_compatible_provider_info(
|
| 568 |
+
api_base, api_key
|
| 569 |
+
)
|
| 570 |
+
elif custom_llm_provider == "bedrock_mantle":
|
| 571 |
+
(
|
| 572 |
+
api_base,
|
| 573 |
+
dynamic_api_key,
|
| 574 |
+
) = litellm.BedrockMantleChatConfig()._get_openai_compatible_provider_info(
|
| 575 |
+
api_base, api_key
|
| 576 |
+
)
|
| 577 |
+
elif custom_llm_provider == "nvidia_nim":
|
| 578 |
+
# nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
|
| 579 |
+
api_base = (
|
| 580 |
+
api_base
|
| 581 |
+
or get_secret("NVIDIA_NIM_API_BASE")
|
| 582 |
+
or "https://integrate.api.nvidia.com/v1"
|
| 583 |
+
) # type: ignore
|
| 584 |
+
dynamic_api_key = api_key or get_secret_str("NVIDIA_NIM_API_KEY")
|
| 585 |
+
elif custom_llm_provider == "cerebras":
|
| 586 |
+
api_base = (
|
| 587 |
+
api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1"
|
| 588 |
+
) # type: ignore
|
| 589 |
+
dynamic_api_key = api_key or get_secret_str("CEREBRAS_API_KEY")
|
| 590 |
+
elif custom_llm_provider == "baseten":
|
| 591 |
+
# Use BasetenConfig to determine the appropriate API base URL
|
| 592 |
+
if api_base is None:
|
| 593 |
+
api_base = litellm.BasetenConfig.get_api_base_for_model(model)
|
| 594 |
+
else:
|
| 595 |
+
api_base = (
|
| 596 |
+
api_base
|
| 597 |
+
or get_secret_str("BASETEN_API_BASE")
|
| 598 |
+
or "https://inference.baseten.co/v1"
|
| 599 |
+
)
|
| 600 |
+
dynamic_api_key = api_key or get_secret_str("BASETEN_API_KEY")
|
| 601 |
+
elif custom_llm_provider == "sambanova":
|
| 602 |
+
api_base = (
|
| 603 |
+
api_base
|
| 604 |
+
or get_secret("SAMBANOVA_API_BASE")
|
| 605 |
+
or "https://api.sambanova.ai/v1"
|
| 606 |
+
) # type: ignore
|
| 607 |
+
dynamic_api_key = api_key or get_secret_str("SAMBANOVA_API_KEY")
|
| 608 |
+
elif custom_llm_provider == "meta_llama":
|
| 609 |
+
api_base = (
|
| 610 |
+
api_base
|
| 611 |
+
or get_secret("LLAMA_API_BASE")
|
| 612 |
+
or "https://api.llama.com/compat/v1"
|
| 613 |
+
) # type: ignore
|
| 614 |
+
dynamic_api_key = api_key or get_secret_str("LLAMA_API_KEY")
|
| 615 |
+
elif custom_llm_provider == "nebius":
|
| 616 |
+
api_base = (
|
| 617 |
+
api_base
|
| 618 |
+
or get_secret("NEBIUS_API_BASE")
|
| 619 |
+
or "https://api.studio.nebius.ai/v1"
|
| 620 |
+
) # type: ignore
|
| 621 |
+
dynamic_api_key = api_key or get_secret_str("NEBIUS_API_KEY")
|
| 622 |
+
elif custom_llm_provider == "ollama":
|
| 623 |
+
api_base = (
|
| 624 |
+
api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434"
|
| 625 |
+
) # type: ignore
|
| 626 |
+
dynamic_api_key = api_key or get_secret_str("OLLAMA_API_KEY")
|
| 627 |
+
elif (custom_llm_provider == "ai21_chat") or (
|
| 628 |
+
custom_llm_provider == "ai21" and model in litellm.ai21_chat_models
|
| 629 |
+
):
|
| 630 |
+
api_base = (
|
| 631 |
+
api_base or get_secret("AI21_API_BASE") or "https://api.ai21.com/studio/v1"
|
| 632 |
+
) # type: ignore
|
| 633 |
+
dynamic_api_key = api_key or get_secret_str("AI21_API_KEY")
|
| 634 |
+
custom_llm_provider = "ai21_chat"
|
| 635 |
+
elif custom_llm_provider == "volcengine":
|
| 636 |
+
# volcengine is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
|
| 637 |
+
api_base = (
|
| 638 |
+
api_base
|
| 639 |
+
or get_secret("VOLCENGINE_API_BASE")
|
| 640 |
+
or "https://ark.cn-beijing.volces.com/api/v3"
|
| 641 |
+
) # type: ignore
|
| 642 |
+
dynamic_api_key = api_key or get_secret_str("VOLCENGINE_API_KEY")
|
| 643 |
+
elif custom_llm_provider == "codestral":
|
| 644 |
+
# codestral is openai compatible, we just need to set this to custom_openai and have the api_base be https://codestral.mistral.ai/v1
|
| 645 |
+
api_base = (
|
| 646 |
+
api_base
|
| 647 |
+
or get_secret("CODESTRAL_API_BASE")
|
| 648 |
+
or "https://codestral.mistral.ai/v1"
|
| 649 |
+
) # type: ignore
|
| 650 |
+
dynamic_api_key = api_key or get_secret_str("CODESTRAL_API_KEY")
|
| 651 |
+
elif custom_llm_provider == "hosted_vllm":
|
| 652 |
+
# vllm is openai compatible, we just need to set this to custom_openai
|
| 653 |
+
(
|
| 654 |
+
api_base,
|
| 655 |
+
dynamic_api_key,
|
| 656 |
+
) = litellm.HostedVLLMChatConfig()._get_openai_compatible_provider_info(
|
| 657 |
+
api_base, api_key
|
| 658 |
+
)
|
| 659 |
+
elif custom_llm_provider == "llamafile":
|
| 660 |
+
# llamafile is OpenAI compatible.
|
| 661 |
+
(
|
| 662 |
+
api_base,
|
| 663 |
+
dynamic_api_key,
|
| 664 |
+
) = litellm.LlamafileChatConfig()._get_openai_compatible_provider_info(
|
| 665 |
+
api_base, api_key
|
| 666 |
+
)
|
| 667 |
+
elif custom_llm_provider == "datarobot":
|
| 668 |
+
# DataRobot is OpenAI compatible.
|
| 669 |
+
(
|
| 670 |
+
api_base,
|
| 671 |
+
dynamic_api_key,
|
| 672 |
+
) = litellm.DataRobotConfig()._get_openai_compatible_provider_info(
|
| 673 |
+
api_base, api_key
|
| 674 |
+
)
|
| 675 |
+
elif custom_llm_provider == "lm_studio":
|
| 676 |
+
# lm_studio is openai compatible, we just need to set this to custom_openai
|
| 677 |
+
(
|
| 678 |
+
api_base,
|
| 679 |
+
dynamic_api_key,
|
| 680 |
+
) = litellm.LMStudioChatConfig()._get_openai_compatible_provider_info(
|
| 681 |
+
api_base, api_key
|
| 682 |
+
)
|
| 683 |
+
elif custom_llm_provider == "deepseek":
|
| 684 |
+
# deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1
|
| 685 |
+
api_base = (
|
| 686 |
+
api_base
|
| 687 |
+
or get_secret("DEEPSEEK_API_BASE")
|
| 688 |
+
or "https://api.deepseek.com/beta"
|
| 689 |
+
) # type: ignore
|
| 690 |
+
|
| 691 |
+
dynamic_api_key = api_key or get_secret_str("DEEPSEEK_API_KEY")
|
| 692 |
+
elif custom_llm_provider == "fireworks_ai":
|
| 693 |
+
# fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.fireworks.ai/inference/v1
|
| 694 |
+
(
|
| 695 |
+
api_base,
|
| 696 |
+
dynamic_api_key,
|
| 697 |
+
) = litellm.FireworksAIConfig()._get_openai_compatible_provider_info(
|
| 698 |
+
api_base=api_base, api_key=api_key
|
| 699 |
+
)
|
| 700 |
+
elif custom_llm_provider == "azure_ai":
|
| 701 |
+
(
|
| 702 |
+
api_base,
|
| 703 |
+
dynamic_api_key,
|
| 704 |
+
custom_llm_provider,
|
| 705 |
+
) = litellm.AzureAIStudioConfig()._get_openai_compatible_provider_info(
|
| 706 |
+
model, api_base, api_key, custom_llm_provider
|
| 707 |
+
)
|
| 708 |
+
elif custom_llm_provider == "github":
|
| 709 |
+
api_base = (
|
| 710 |
+
api_base
|
| 711 |
+
or get_secret_str("GITHUB_API_BASE")
|
| 712 |
+
or "https://models.inference.ai.azure.com" # This is github's default base url
|
| 713 |
+
)
|
| 714 |
+
dynamic_api_key = api_key or get_secret_str("GITHUB_API_KEY")
|
| 715 |
+
elif custom_llm_provider == "litellm_proxy":
|
| 716 |
+
(
|
| 717 |
+
api_base,
|
| 718 |
+
dynamic_api_key,
|
| 719 |
+
) = litellm.LiteLLMProxyChatConfig()._get_openai_compatible_provider_info(
|
| 720 |
+
api_base=api_base, api_key=api_key
|
| 721 |
+
)
|
| 722 |
+
|
| 723 |
+
elif custom_llm_provider == "mistral":
|
| 724 |
+
(
|
| 725 |
+
api_base,
|
| 726 |
+
dynamic_api_key,
|
| 727 |
+
) = litellm.MistralConfig()._get_openai_compatible_provider_info(
|
| 728 |
+
api_base, api_key
|
| 729 |
+
)
|
| 730 |
+
elif custom_llm_provider == "jina_ai":
|
| 731 |
+
(
|
| 732 |
+
custom_llm_provider,
|
| 733 |
+
api_base,
|
| 734 |
+
dynamic_api_key,
|
| 735 |
+
) = litellm.JinaAIEmbeddingConfig()._get_openai_compatible_provider_info(
|
| 736 |
+
api_base, api_key
|
| 737 |
+
)
|
| 738 |
+
elif custom_llm_provider == "xai":
|
| 739 |
+
(
|
| 740 |
+
api_base,
|
| 741 |
+
dynamic_api_key,
|
| 742 |
+
) = litellm.XAIChatConfig()._get_openai_compatible_provider_info(
|
| 743 |
+
api_base, api_key
|
| 744 |
+
)
|
| 745 |
+
elif custom_llm_provider == "zai":
|
| 746 |
+
(
|
| 747 |
+
api_base,
|
| 748 |
+
dynamic_api_key,
|
| 749 |
+
) = litellm.ZAIChatConfig()._get_openai_compatible_provider_info(
|
| 750 |
+
api_base, api_key
|
| 751 |
+
)
|
| 752 |
+
elif custom_llm_provider == "together_ai":
|
| 753 |
+
api_base = (
|
| 754 |
+
api_base
|
| 755 |
+
or get_secret_str("TOGETHER_AI_API_BASE")
|
| 756 |
+
or "https://api.together.xyz/v1"
|
| 757 |
+
) # type: ignore
|
| 758 |
+
dynamic_api_key = api_key or (
|
| 759 |
+
get_secret_str("TOGETHER_API_KEY")
|
| 760 |
+
or get_secret_str("TOGETHER_AI_API_KEY")
|
| 761 |
+
or get_secret_str("TOGETHERAI_API_KEY")
|
| 762 |
+
or get_secret_str("TOGETHER_AI_TOKEN")
|
| 763 |
+
)
|
| 764 |
+
elif custom_llm_provider == "friendliai":
|
| 765 |
+
api_base = (
|
| 766 |
+
api_base
|
| 767 |
+
or get_secret("FRIENDLI_API_BASE")
|
| 768 |
+
or "https://api.friendli.ai/serverless/v1"
|
| 769 |
+
) # type: ignore
|
| 770 |
+
dynamic_api_key = (
|
| 771 |
+
api_key
|
| 772 |
+
or get_secret_str("FRIENDLIAI_API_KEY")
|
| 773 |
+
or get_secret_str("FRIENDLI_TOKEN")
|
| 774 |
+
)
|
| 775 |
+
elif custom_llm_provider == "galadriel":
|
| 776 |
+
api_base = (
|
| 777 |
+
api_base
|
| 778 |
+
or get_secret("GALADRIEL_API_BASE")
|
| 779 |
+
or "https://api.galadriel.com/v1"
|
| 780 |
+
) # type: ignore
|
| 781 |
+
dynamic_api_key = api_key or get_secret_str("GALADRIEL_API_KEY")
|
| 782 |
+
elif custom_llm_provider == "github_copilot":
|
| 783 |
+
(
|
| 784 |
+
api_base,
|
| 785 |
+
dynamic_api_key,
|
| 786 |
+
custom_llm_provider,
|
| 787 |
+
) = litellm.GithubCopilotConfig()._get_openai_compatible_provider_info(
|
| 788 |
+
model, api_base, api_key, custom_llm_provider
|
| 789 |
+
)
|
| 790 |
+
elif custom_llm_provider == "chatgpt":
|
| 791 |
+
(
|
| 792 |
+
api_base,
|
| 793 |
+
dynamic_api_key,
|
| 794 |
+
custom_llm_provider,
|
| 795 |
+
) = litellm.ChatGPTConfig()._get_openai_compatible_provider_info(
|
| 796 |
+
model, api_base, api_key, custom_llm_provider
|
| 797 |
+
)
|
| 798 |
+
elif custom_llm_provider == "novita":
|
| 799 |
+
api_base = (
|
| 800 |
+
api_base
|
| 801 |
+
or get_secret("NOVITA_API_BASE")
|
| 802 |
+
or "https://api.novita.ai/v3/openai"
|
| 803 |
+
) # type: ignore
|
| 804 |
+
dynamic_api_key = api_key or get_secret_str("NOVITA_API_KEY")
|
| 805 |
+
elif custom_llm_provider == "snowflake":
|
| 806 |
+
(
|
| 807 |
+
api_base,
|
| 808 |
+
dynamic_api_key,
|
| 809 |
+
) = litellm.SnowflakeConfig()._get_openai_compatible_provider_info(
|
| 810 |
+
api_base, api_key
|
| 811 |
+
)
|
| 812 |
+
elif custom_llm_provider == "gradient_ai":
|
| 813 |
+
(
|
| 814 |
+
api_base,
|
| 815 |
+
dynamic_api_key,
|
| 816 |
+
) = litellm.GradientAIConfig()._get_openai_compatible_provider_info(
|
| 817 |
+
api_base, api_key
|
| 818 |
+
)
|
| 819 |
+
elif custom_llm_provider == "featherless_ai":
|
| 820 |
+
(
|
| 821 |
+
api_base,
|
| 822 |
+
dynamic_api_key,
|
| 823 |
+
) = litellm.FeatherlessAIConfig()._get_openai_compatible_provider_info(
|
| 824 |
+
api_base, api_key
|
| 825 |
+
)
|
| 826 |
+
elif custom_llm_provider == "nscale":
|
| 827 |
+
(
|
| 828 |
+
api_base,
|
| 829 |
+
dynamic_api_key,
|
| 830 |
+
) = litellm.NscaleConfig()._get_openai_compatible_provider_info(
|
| 831 |
+
api_base=api_base, api_key=api_key
|
| 832 |
+
)
|
| 833 |
+
elif custom_llm_provider == "heroku":
|
| 834 |
+
(
|
| 835 |
+
api_base,
|
| 836 |
+
dynamic_api_key,
|
| 837 |
+
) = litellm.HerokuChatConfig()._get_openai_compatible_provider_info(
|
| 838 |
+
api_base, api_key
|
| 839 |
+
)
|
| 840 |
+
elif custom_llm_provider == "dashscope":
|
| 841 |
+
(
|
| 842 |
+
api_base,
|
| 843 |
+
dynamic_api_key,
|
| 844 |
+
) = litellm.DashScopeChatConfig()._get_openai_compatible_provider_info(
|
| 845 |
+
api_base, api_key
|
| 846 |
+
)
|
| 847 |
+
elif custom_llm_provider == "moonshot":
|
| 848 |
+
(
|
| 849 |
+
api_base,
|
| 850 |
+
dynamic_api_key,
|
| 851 |
+
) = litellm.MoonshotChatConfig()._get_openai_compatible_provider_info(
|
| 852 |
+
api_base, api_key
|
| 853 |
+
)
|
| 854 |
+
# publicai is now handled by JSON config (see litellm/llms/openai_like/providers.json)
|
| 855 |
+
elif custom_llm_provider == "docker_model_runner":
|
| 856 |
+
(
|
| 857 |
+
api_base,
|
| 858 |
+
dynamic_api_key,
|
| 859 |
+
) = litellm.DockerModelRunnerChatConfig()._get_openai_compatible_provider_info(
|
| 860 |
+
api_base, api_key
|
| 861 |
+
)
|
| 862 |
+
elif custom_llm_provider == "v0":
|
| 863 |
+
(
|
| 864 |
+
api_base,
|
| 865 |
+
dynamic_api_key,
|
| 866 |
+
) = litellm.V0ChatConfig()._get_openai_compatible_provider_info(
|
| 867 |
+
api_base, api_key
|
| 868 |
+
)
|
| 869 |
+
elif custom_llm_provider == "morph":
|
| 870 |
+
(
|
| 871 |
+
api_base,
|
| 872 |
+
dynamic_api_key,
|
| 873 |
+
) = litellm.MorphChatConfig()._get_openai_compatible_provider_info(
|
| 874 |
+
api_base, api_key
|
| 875 |
+
)
|
| 876 |
+
elif custom_llm_provider == "lambda_ai":
|
| 877 |
+
(
|
| 878 |
+
api_base,
|
| 879 |
+
dynamic_api_key,
|
| 880 |
+
) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info(
|
| 881 |
+
api_base, api_key
|
| 882 |
+
)
|
| 883 |
+
elif custom_llm_provider == "hyperbolic":
|
| 884 |
+
(
|
| 885 |
+
api_base,
|
| 886 |
+
dynamic_api_key,
|
| 887 |
+
) = litellm.HyperbolicChatConfig()._get_openai_compatible_provider_info(
|
| 888 |
+
api_base, api_key
|
| 889 |
+
)
|
| 890 |
+
elif custom_llm_provider == "vercel_ai_gateway":
|
| 891 |
+
(
|
| 892 |
+
api_base,
|
| 893 |
+
dynamic_api_key,
|
| 894 |
+
) = litellm.VercelAIGatewayConfig()._get_openai_compatible_provider_info(
|
| 895 |
+
api_base, api_key
|
| 896 |
+
)
|
| 897 |
+
elif custom_llm_provider == "aiml":
|
| 898 |
+
(
|
| 899 |
+
api_base,
|
| 900 |
+
dynamic_api_key,
|
| 901 |
+
) = litellm.AIMLChatConfig()._get_openai_compatible_provider_info(
|
| 902 |
+
api_base, api_key
|
| 903 |
+
)
|
| 904 |
+
elif custom_llm_provider == "wandb":
|
| 905 |
+
api_base = (
|
| 906 |
+
api_base
|
| 907 |
+
or get_secret("WANDB_API_BASE")
|
| 908 |
+
or "https://api.inference.wandb.ai/v1"
|
| 909 |
+
) # type: ignore
|
| 910 |
+
dynamic_api_key = api_key or get_secret_str("WANDB_API_KEY")
|
| 911 |
+
elif custom_llm_provider == "lemonade":
|
| 912 |
+
(
|
| 913 |
+
api_base,
|
| 914 |
+
dynamic_api_key,
|
| 915 |
+
) = litellm.LemonadeChatConfig()._get_openai_compatible_provider_info(
|
| 916 |
+
api_base, api_key
|
| 917 |
+
)
|
| 918 |
+
elif custom_llm_provider == "clarifai":
|
| 919 |
+
(
|
| 920 |
+
api_base,
|
| 921 |
+
dynamic_api_key,
|
| 922 |
+
) = litellm.ClarifaiConfig()._get_openai_compatible_provider_info(
|
| 923 |
+
api_base, api_key
|
| 924 |
+
)
|
| 925 |
+
elif custom_llm_provider == "ragflow":
|
| 926 |
+
full_model = f"ragflow/{model}"
|
| 927 |
+
(
|
| 928 |
+
api_base,
|
| 929 |
+
dynamic_api_key,
|
| 930 |
+
_,
|
| 931 |
+
) = litellm.RAGFlowConfig()._get_openai_compatible_provider_info(
|
| 932 |
+
full_model, api_base, api_key, "ragflow"
|
| 933 |
+
)
|
| 934 |
+
model = full_model
|
| 935 |
+
elif custom_llm_provider == "langgraph":
|
| 936 |
+
# LangGraph is a custom provider, just need to set api_base
|
| 937 |
+
api_base = (
|
| 938 |
+
api_base or get_secret_str("LANGGRAPH_API_BASE") or "http://localhost:2024"
|
| 939 |
+
)
|
| 940 |
+
dynamic_api_key = api_key or get_secret_str("LANGGRAPH_API_KEY")
|
| 941 |
+
elif custom_llm_provider == "manus":
|
| 942 |
+
# Manus is OpenAI compatible for responses API
|
| 943 |
+
api_base = (
|
| 944 |
+
api_base or get_secret_str("MANUS_API_BASE") or "https://api.manus.im"
|
| 945 |
+
)
|
| 946 |
+
dynamic_api_key = api_key or get_secret_str("MANUS_API_KEY")
|
| 947 |
+
|
| 948 |
+
if api_base is not None and not isinstance(api_base, str):
|
| 949 |
+
raise Exception("api base needs to be a string. api_base={}".format(api_base))
|
| 950 |
+
if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
|
| 951 |
+
raise Exception(
|
| 952 |
+
"dynamic_api_key needs to be a string. dynamic_api_key={}".format(
|
| 953 |
+
dynamic_api_key
|
| 954 |
+
)
|
| 955 |
+
)
|
| 956 |
+
if dynamic_api_key is None and api_key is not None:
|
| 957 |
+
dynamic_api_key = api_key
|
| 958 |
+
return model, custom_llm_provider, dynamic_api_key, api_base
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_model_cost_map.py
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Pulls the cost + context window + provider route for known models from https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json
|
| 3 |
+
|
| 4 |
+
This can be disabled by setting the LITELLM_LOCAL_MODEL_COST_MAP environment variable to True.
|
| 5 |
+
|
| 6 |
+
```
|
| 7 |
+
export LITELLM_LOCAL_MODEL_COST_MAP=True
|
| 8 |
+
```
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import json
|
| 12 |
+
import os
|
| 13 |
+
from importlib.resources import files
|
| 14 |
+
from typing import Dict, List, Optional
|
| 15 |
+
|
| 16 |
+
import httpx
|
| 17 |
+
|
| 18 |
+
from litellm import verbose_logger
|
| 19 |
+
from litellm.constants import (
|
| 20 |
+
MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
| 21 |
+
MODEL_COST_MAP_MIN_MODEL_COUNT,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class GetModelCostMap:
|
| 26 |
+
"""
|
| 27 |
+
Handles fetching, validating, and loading the model cost map.
|
| 28 |
+
|
| 29 |
+
Only the backup model *count* is cached (a single int). The full
|
| 30 |
+
backup dict is never held in memory — it is only parsed when it
|
| 31 |
+
needs to be *returned* as a fallback.
|
| 32 |
+
"""
|
| 33 |
+
|
| 34 |
+
_backup_model_count: int = -1 # -1 = not yet loaded
|
| 35 |
+
|
| 36 |
+
@staticmethod
|
| 37 |
+
def load_local_model_cost_map() -> dict:
|
| 38 |
+
"""Load the local backup model cost map bundled with the package."""
|
| 39 |
+
content = json.loads(
|
| 40 |
+
files("litellm")
|
| 41 |
+
.joinpath("model_prices_and_context_window_backup.json")
|
| 42 |
+
.read_text(encoding="utf-8")
|
| 43 |
+
)
|
| 44 |
+
return content
|
| 45 |
+
|
| 46 |
+
@classmethod
|
| 47 |
+
def _get_backup_model_count(cls) -> int:
|
| 48 |
+
"""Return the number of models in the local backup (cached int)."""
|
| 49 |
+
if cls._backup_model_count < 0:
|
| 50 |
+
backup = cls.load_local_model_cost_map()
|
| 51 |
+
cls._backup_model_count = len(backup)
|
| 52 |
+
return cls._backup_model_count
|
| 53 |
+
|
| 54 |
+
@staticmethod
|
| 55 |
+
def _check_is_valid_dict(fetched_map: dict) -> bool:
|
| 56 |
+
"""Check 1: fetched map is a non-empty dict."""
|
| 57 |
+
if not isinstance(fetched_map, dict):
|
| 58 |
+
verbose_logger.warning(
|
| 59 |
+
"LiteLLM: Fetched model cost map is not a dict (type=%s). "
|
| 60 |
+
"Falling back to local backup.",
|
| 61 |
+
type(fetched_map).__name__,
|
| 62 |
+
)
|
| 63 |
+
return False
|
| 64 |
+
|
| 65 |
+
if len(fetched_map) == 0:
|
| 66 |
+
verbose_logger.warning(
|
| 67 |
+
"LiteLLM: Fetched model cost map is empty. "
|
| 68 |
+
"Falling back to local backup.",
|
| 69 |
+
)
|
| 70 |
+
return False
|
| 71 |
+
|
| 72 |
+
return True
|
| 73 |
+
|
| 74 |
+
@classmethod
|
| 75 |
+
def _check_model_count_not_reduced(
|
| 76 |
+
cls,
|
| 77 |
+
fetched_map: dict,
|
| 78 |
+
backup_model_count: int,
|
| 79 |
+
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
|
| 80 |
+
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
| 81 |
+
) -> bool:
|
| 82 |
+
"""Check 2: model count has not reduced significantly vs backup."""
|
| 83 |
+
fetched_count = len(fetched_map)
|
| 84 |
+
|
| 85 |
+
if fetched_count < min_model_count:
|
| 86 |
+
verbose_logger.warning(
|
| 87 |
+
"LiteLLM: Fetched model cost map has only %d models (minimum=%d). "
|
| 88 |
+
"This may indicate a corrupted upstream file. "
|
| 89 |
+
"Falling back to local backup.",
|
| 90 |
+
fetched_count,
|
| 91 |
+
min_model_count,
|
| 92 |
+
)
|
| 93 |
+
return False
|
| 94 |
+
|
| 95 |
+
if (
|
| 96 |
+
backup_model_count > 0
|
| 97 |
+
and fetched_count < backup_model_count * max_shrink_ratio
|
| 98 |
+
):
|
| 99 |
+
verbose_logger.warning(
|
| 100 |
+
"LiteLLM: Fetched model cost map shrank significantly "
|
| 101 |
+
"(fetched=%d, backup=%d, threshold=%.0f%%). "
|
| 102 |
+
"This may indicate a corrupted upstream file. "
|
| 103 |
+
"Falling back to local backup.",
|
| 104 |
+
fetched_count,
|
| 105 |
+
backup_model_count,
|
| 106 |
+
max_shrink_ratio * 100,
|
| 107 |
+
)
|
| 108 |
+
return False
|
| 109 |
+
|
| 110 |
+
return True
|
| 111 |
+
|
| 112 |
+
@classmethod
|
| 113 |
+
def validate_model_cost_map(
|
| 114 |
+
cls,
|
| 115 |
+
fetched_map: dict,
|
| 116 |
+
backup_model_count: int,
|
| 117 |
+
min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
|
| 118 |
+
max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
|
| 119 |
+
) -> bool:
|
| 120 |
+
"""
|
| 121 |
+
Validate the integrity of a fetched model cost map.
|
| 122 |
+
|
| 123 |
+
Runs each check in order and returns False on the first failure.
|
| 124 |
+
|
| 125 |
+
Checks:
|
| 126 |
+
1. ``_check_is_valid_dict`` -- fetched map is a non-empty dict.
|
| 127 |
+
2. ``_check_model_count_not_reduced`` -- model count meets minimum
|
| 128 |
+
and has not shrunk >``max_shrink_ratio`` vs backup.
|
| 129 |
+
|
| 130 |
+
Returns True if all checks pass, False otherwise.
|
| 131 |
+
"""
|
| 132 |
+
if not cls._check_is_valid_dict(fetched_map):
|
| 133 |
+
return False
|
| 134 |
+
|
| 135 |
+
if not cls._check_model_count_not_reduced(
|
| 136 |
+
fetched_map=fetched_map,
|
| 137 |
+
backup_model_count=backup_model_count,
|
| 138 |
+
min_model_count=min_model_count,
|
| 139 |
+
max_shrink_ratio=max_shrink_ratio,
|
| 140 |
+
):
|
| 141 |
+
return False
|
| 142 |
+
|
| 143 |
+
return True
|
| 144 |
+
|
| 145 |
+
@staticmethod
|
| 146 |
+
def fetch_remote_model_cost_map(url: str, timeout: int = 5) -> dict:
|
| 147 |
+
"""
|
| 148 |
+
Fetch the model cost map from a remote URL.
|
| 149 |
+
|
| 150 |
+
Returns the parsed JSON dict. Raises on network/parse errors
|
| 151 |
+
(caller is expected to handle).
|
| 152 |
+
"""
|
| 153 |
+
response = httpx.get(url, timeout=timeout)
|
| 154 |
+
response.raise_for_status()
|
| 155 |
+
return response.json()
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
class ModelCostMapSourceInfo:
|
| 159 |
+
"""Tracks the source of the currently loaded model cost map."""
|
| 160 |
+
|
| 161 |
+
source: str = "local" # "local" or "remote"
|
| 162 |
+
url: Optional[str] = None
|
| 163 |
+
is_env_forced: bool = False
|
| 164 |
+
fallback_reason: Optional[str] = None
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
# Module-level singleton tracking the source of the current cost map
|
| 168 |
+
_cost_map_source_info = ModelCostMapSourceInfo()
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def get_model_cost_map_source_info() -> dict:
|
| 172 |
+
"""
|
| 173 |
+
Return metadata about where the current model cost map was loaded from.
|
| 174 |
+
|
| 175 |
+
Returns a dict with:
|
| 176 |
+
- source: "local" or "remote"
|
| 177 |
+
- url: the remote URL attempted (or None for local-only)
|
| 178 |
+
- is_env_forced: True if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage
|
| 179 |
+
- fallback_reason: human-readable reason if remote failed and local was used
|
| 180 |
+
"""
|
| 181 |
+
return {
|
| 182 |
+
"source": _cost_map_source_info.source,
|
| 183 |
+
"url": _cost_map_source_info.url,
|
| 184 |
+
"is_env_forced": _cost_map_source_info.is_env_forced,
|
| 185 |
+
"fallback_reason": _cost_map_source_info.fallback_reason,
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def _expand_model_aliases(model_cost: dict) -> dict:
|
| 190 |
+
"""
|
| 191 |
+
Expand ``aliases`` lists in model cost entries into top-level entries.
|
| 192 |
+
|
| 193 |
+
Each alias gets a reference to the **same** dict object as the canonical
|
| 194 |
+
entry (zero memory overhead). The ``aliases`` key is removed from the
|
| 195 |
+
entry so downstream code never sees it.
|
| 196 |
+
|
| 197 |
+
If an alias collides with an existing canonical entry the alias is
|
| 198 |
+
skipped and a warning is logged.
|
| 199 |
+
"""
|
| 200 |
+
aliases_to_add: Dict[str, dict] = {}
|
| 201 |
+
keys_with_aliases: List[str] = []
|
| 202 |
+
|
| 203 |
+
for model_name, model_info in model_cost.items():
|
| 204 |
+
aliases: Optional[list] = model_info.get("aliases")
|
| 205 |
+
if aliases is None:
|
| 206 |
+
continue
|
| 207 |
+
keys_with_aliases.append(model_name)
|
| 208 |
+
if not isinstance(aliases, list):
|
| 209 |
+
verbose_logger.warning(
|
| 210 |
+
"LiteLLM model alias field for '%s' is not a list (got %s) — skipping.",
|
| 211 |
+
model_name,
|
| 212 |
+
type(aliases).__name__,
|
| 213 |
+
)
|
| 214 |
+
continue
|
| 215 |
+
if not aliases:
|
| 216 |
+
continue
|
| 217 |
+
for alias in aliases:
|
| 218 |
+
if alias in model_cost:
|
| 219 |
+
verbose_logger.warning(
|
| 220 |
+
"LiteLLM model alias conflict: alias '%s' (from '%s') "
|
| 221 |
+
"already exists as a canonical entry — skipping.",
|
| 222 |
+
alias,
|
| 223 |
+
model_name,
|
| 224 |
+
)
|
| 225 |
+
continue
|
| 226 |
+
if alias in aliases_to_add:
|
| 227 |
+
verbose_logger.warning(
|
| 228 |
+
"LiteLLM model alias conflict: alias '%s' (from '%s') "
|
| 229 |
+
"was already claimed by another entry — skipping.",
|
| 230 |
+
alias,
|
| 231 |
+
model_name,
|
| 232 |
+
)
|
| 233 |
+
continue
|
| 234 |
+
aliases_to_add[alias] = model_info # same dict reference
|
| 235 |
+
|
| 236 |
+
# Remove the ``aliases`` key from entries so it doesn't pollute model info
|
| 237 |
+
for key in keys_with_aliases:
|
| 238 |
+
model_cost[key].pop("aliases", None)
|
| 239 |
+
|
| 240 |
+
model_cost.update(aliases_to_add)
|
| 241 |
+
return model_cost
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def get_model_cost_map(url: str) -> dict:
|
| 245 |
+
"""
|
| 246 |
+
Public entry point — returns the model cost map dict.
|
| 247 |
+
|
| 248 |
+
1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only.
|
| 249 |
+
2. Otherwise fetches from ``url``, validates integrity, and falls back
|
| 250 |
+
to the local backup on any failure.
|
| 251 |
+
|
| 252 |
+
Only the backup model count is cached (a single int) for validation.
|
| 253 |
+
The full backup dict is only parsed when it must be *returned* as a
|
| 254 |
+
fallback — it is never held in memory long-term.
|
| 255 |
+
"""
|
| 256 |
+
# Note: can't use get_secret_bool here — this runs during litellm.__init__
|
| 257 |
+
# before litellm._key_management_settings is set.
|
| 258 |
+
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
|
| 259 |
+
_cost_map_source_info.source = "local"
|
| 260 |
+
_cost_map_source_info.url = None
|
| 261 |
+
_cost_map_source_info.is_env_forced = True
|
| 262 |
+
_cost_map_source_info.fallback_reason = None
|
| 263 |
+
return _expand_model_aliases(GetModelCostMap.load_local_model_cost_map())
|
| 264 |
+
|
| 265 |
+
_cost_map_source_info.url = url
|
| 266 |
+
_cost_map_source_info.is_env_forced = False
|
| 267 |
+
|
| 268 |
+
try:
|
| 269 |
+
content = GetModelCostMap.fetch_remote_model_cost_map(url)
|
| 270 |
+
except Exception as e:
|
| 271 |
+
verbose_logger.warning(
|
| 272 |
+
"LiteLLM: Failed to fetch remote model cost map from %s: %s. "
|
| 273 |
+
"Falling back to local backup.",
|
| 274 |
+
url,
|
| 275 |
+
str(e),
|
| 276 |
+
)
|
| 277 |
+
_cost_map_source_info.source = "local"
|
| 278 |
+
_cost_map_source_info.fallback_reason = f"Remote fetch failed: {str(e)}"
|
| 279 |
+
return _expand_model_aliases(GetModelCostMap.load_local_model_cost_map())
|
| 280 |
+
|
| 281 |
+
# Validate using cached count (cheap int comparison, no file I/O)
|
| 282 |
+
if not GetModelCostMap.validate_model_cost_map(
|
| 283 |
+
fetched_map=content,
|
| 284 |
+
backup_model_count=GetModelCostMap._get_backup_model_count(),
|
| 285 |
+
):
|
| 286 |
+
verbose_logger.warning(
|
| 287 |
+
"LiteLLM: Fetched model cost map failed integrity check. "
|
| 288 |
+
"Using local backup instead. url=%s",
|
| 289 |
+
url,
|
| 290 |
+
)
|
| 291 |
+
_cost_map_source_info.source = "local"
|
| 292 |
+
_cost_map_source_info.fallback_reason = (
|
| 293 |
+
"Remote data failed integrity validation"
|
| 294 |
+
)
|
| 295 |
+
return _expand_model_aliases(GetModelCostMap.load_local_model_cost_map())
|
| 296 |
+
|
| 297 |
+
_cost_map_source_info.source = "remote"
|
| 298 |
+
_cost_map_source_info.fallback_reason = None
|
| 299 |
+
return _expand_model_aliases(content)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_provider_specific_headers.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, Optional
|
| 2 |
+
|
| 3 |
+
from litellm.types.utils import ProviderSpecificHeader
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class ProviderSpecificHeaderUtils:
|
| 7 |
+
@staticmethod
|
| 8 |
+
def get_provider_specific_headers(
|
| 9 |
+
provider_specific_header: Optional[ProviderSpecificHeader],
|
| 10 |
+
custom_llm_provider: Optional[str],
|
| 11 |
+
) -> Dict:
|
| 12 |
+
"""
|
| 13 |
+
Get the provider specific headers for the given custom llm provider.
|
| 14 |
+
|
| 15 |
+
Supports comma-separated provider lists for headers that work across multiple providers.
|
| 16 |
+
|
| 17 |
+
Returns:
|
| 18 |
+
Dict: The provider specific headers for the given custom llm provider
|
| 19 |
+
"""
|
| 20 |
+
if provider_specific_header is None or custom_llm_provider is None:
|
| 21 |
+
return {}
|
| 22 |
+
|
| 23 |
+
stored_providers = provider_specific_header.get("custom_llm_provider", "")
|
| 24 |
+
provider_list = [p.strip() for p in stored_providers.split(",")]
|
| 25 |
+
|
| 26 |
+
if custom_llm_provider in provider_list:
|
| 27 |
+
return provider_specific_header.get("extra_headers", {})
|
| 28 |
+
|
| 29 |
+
return {}
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_supported_openai_params.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Literal, Optional
|
| 2 |
+
|
| 3 |
+
import litellm
|
| 4 |
+
from litellm.exceptions import BadRequestError
|
| 5 |
+
from litellm.types.utils import LlmProviders, LlmProvidersSet
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
def get_supported_openai_params( # noqa: PLR0915
|
| 9 |
+
model: str,
|
| 10 |
+
custom_llm_provider: Optional[str] = None,
|
| 11 |
+
request_type: Literal[
|
| 12 |
+
"chat_completion", "embeddings", "transcription"
|
| 13 |
+
] = "chat_completion",
|
| 14 |
+
) -> Optional[list]:
|
| 15 |
+
"""
|
| 16 |
+
Returns the supported openai params for a given model + provider
|
| 17 |
+
|
| 18 |
+
Example:
|
| 19 |
+
```
|
| 20 |
+
get_supported_openai_params(model="anthropic.claude-3", custom_llm_provider="bedrock")
|
| 21 |
+
```
|
| 22 |
+
|
| 23 |
+
Returns:
|
| 24 |
+
- List if custom_llm_provider is mapped
|
| 25 |
+
- None if unmapped
|
| 26 |
+
"""
|
| 27 |
+
if not custom_llm_provider:
|
| 28 |
+
try:
|
| 29 |
+
custom_llm_provider = litellm.get_llm_provider(model=model)[1]
|
| 30 |
+
except BadRequestError:
|
| 31 |
+
return None
|
| 32 |
+
|
| 33 |
+
if custom_llm_provider in LlmProvidersSet:
|
| 34 |
+
provider_config = litellm.ProviderConfigManager.get_provider_chat_config(
|
| 35 |
+
model=model, provider=LlmProviders(custom_llm_provider)
|
| 36 |
+
)
|
| 37 |
+
elif custom_llm_provider.split("/")[0] in LlmProvidersSet:
|
| 38 |
+
provider_config = litellm.ProviderConfigManager.get_provider_chat_config(
|
| 39 |
+
model=model, provider=LlmProviders(custom_llm_provider.split("/")[0])
|
| 40 |
+
)
|
| 41 |
+
else:
|
| 42 |
+
provider_config = None
|
| 43 |
+
|
| 44 |
+
if provider_config and request_type == "chat_completion":
|
| 45 |
+
return provider_config.get_supported_openai_params(model=model)
|
| 46 |
+
|
| 47 |
+
if custom_llm_provider == "bedrock":
|
| 48 |
+
return litellm.AmazonConverseConfig().get_supported_openai_params(model=model)
|
| 49 |
+
elif custom_llm_provider == "meta_llama":
|
| 50 |
+
provider_config = litellm.ProviderConfigManager.get_provider_chat_config(
|
| 51 |
+
model=model, provider=LlmProviders.LLAMA
|
| 52 |
+
)
|
| 53 |
+
if provider_config:
|
| 54 |
+
return provider_config.get_supported_openai_params(model=model)
|
| 55 |
+
elif custom_llm_provider == "ollama":
|
| 56 |
+
return litellm.OllamaConfig().get_supported_openai_params(model=model)
|
| 57 |
+
elif custom_llm_provider == "ollama_chat":
|
| 58 |
+
return litellm.OllamaChatConfig().get_supported_openai_params(model=model)
|
| 59 |
+
elif custom_llm_provider == "anthropic":
|
| 60 |
+
return litellm.AnthropicConfig().get_supported_openai_params(model=model)
|
| 61 |
+
elif custom_llm_provider == "anthropic_text":
|
| 62 |
+
return litellm.AnthropicTextConfig().get_supported_openai_params(model=model)
|
| 63 |
+
elif custom_llm_provider == "fireworks_ai":
|
| 64 |
+
if request_type == "embeddings":
|
| 65 |
+
return litellm.FireworksAIEmbeddingConfig().get_supported_openai_params(
|
| 66 |
+
model=model
|
| 67 |
+
)
|
| 68 |
+
elif request_type == "transcription":
|
| 69 |
+
return litellm.FireworksAIAudioTranscriptionConfig().get_supported_openai_params(
|
| 70 |
+
model=model
|
| 71 |
+
)
|
| 72 |
+
else:
|
| 73 |
+
return litellm.FireworksAIConfig().get_supported_openai_params(model=model)
|
| 74 |
+
elif custom_llm_provider == "nvidia_nim":
|
| 75 |
+
if request_type == "chat_completion":
|
| 76 |
+
return litellm.nvidiaNimConfig.get_supported_openai_params(model=model)
|
| 77 |
+
elif request_type == "embeddings":
|
| 78 |
+
return litellm.nvidiaNimEmbeddingConfig.get_supported_openai_params()
|
| 79 |
+
elif custom_llm_provider == "cerebras":
|
| 80 |
+
return litellm.CerebrasConfig().get_supported_openai_params(model=model)
|
| 81 |
+
elif custom_llm_provider == "baseten":
|
| 82 |
+
return litellm.BasetenConfig().get_supported_openai_params(model=model)
|
| 83 |
+
elif custom_llm_provider == "xai":
|
| 84 |
+
return litellm.XAIChatConfig().get_supported_openai_params(model=model)
|
| 85 |
+
elif custom_llm_provider == "ai21_chat" or custom_llm_provider == "ai21":
|
| 86 |
+
return litellm.AI21ChatConfig().get_supported_openai_params(model=model)
|
| 87 |
+
elif custom_llm_provider == "volcengine":
|
| 88 |
+
return litellm.VolcEngineConfig().get_supported_openai_params(model=model)
|
| 89 |
+
elif custom_llm_provider == "groq":
|
| 90 |
+
return litellm.GroqChatConfig().get_supported_openai_params(model=model)
|
| 91 |
+
elif custom_llm_provider == "bedrock_mantle":
|
| 92 |
+
return litellm.BedrockMantleChatConfig().get_supported_openai_params(
|
| 93 |
+
model=model
|
| 94 |
+
)
|
| 95 |
+
elif custom_llm_provider == "hosted_vllm":
|
| 96 |
+
return litellm.HostedVLLMChatConfig().get_supported_openai_params(model=model)
|
| 97 |
+
elif custom_llm_provider == "vllm":
|
| 98 |
+
return litellm.VLLMConfig().get_supported_openai_params(model=model)
|
| 99 |
+
elif custom_llm_provider == "deepseek":
|
| 100 |
+
return litellm.DeepSeekChatConfig().get_supported_openai_params(model=model)
|
| 101 |
+
elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere":
|
| 102 |
+
return litellm.CohereChatConfig().get_supported_openai_params(model=model)
|
| 103 |
+
elif custom_llm_provider == "maritalk":
|
| 104 |
+
return litellm.MaritalkConfig().get_supported_openai_params(model=model)
|
| 105 |
+
elif custom_llm_provider == "openai":
|
| 106 |
+
if request_type == "transcription":
|
| 107 |
+
transcription_provider_config = (
|
| 108 |
+
litellm.ProviderConfigManager.get_provider_audio_transcription_config(
|
| 109 |
+
model=model, provider=LlmProviders.OPENAI
|
| 110 |
+
)
|
| 111 |
+
)
|
| 112 |
+
if isinstance(
|
| 113 |
+
transcription_provider_config, litellm.OpenAIGPTAudioTranscriptionConfig
|
| 114 |
+
):
|
| 115 |
+
return transcription_provider_config.get_supported_openai_params(
|
| 116 |
+
model=model
|
| 117 |
+
)
|
| 118 |
+
else:
|
| 119 |
+
raise ValueError(
|
| 120 |
+
f"Unsupported provider config: {transcription_provider_config} for model: {model}"
|
| 121 |
+
)
|
| 122 |
+
return litellm.OpenAIConfig().get_supported_openai_params(model=model)
|
| 123 |
+
elif custom_llm_provider == "sap":
|
| 124 |
+
if request_type == "chat_completion":
|
| 125 |
+
return litellm.GenAIHubOrchestrationConfig().get_supported_openai_params(
|
| 126 |
+
model=model
|
| 127 |
+
)
|
| 128 |
+
elif request_type == "embeddings":
|
| 129 |
+
return litellm.GenAIHubEmbeddingConfig().get_supported_openai_params(
|
| 130 |
+
model=model
|
| 131 |
+
)
|
| 132 |
+
elif custom_llm_provider == "azure":
|
| 133 |
+
if litellm.AzureOpenAIO1Config().is_o_series_model(model=model):
|
| 134 |
+
return litellm.AzureOpenAIO1Config().get_supported_openai_params(
|
| 135 |
+
model=model
|
| 136 |
+
)
|
| 137 |
+
elif litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model=model):
|
| 138 |
+
return litellm.AzureOpenAIGPT5Config().get_supported_openai_params(
|
| 139 |
+
model=model
|
| 140 |
+
)
|
| 141 |
+
else:
|
| 142 |
+
return litellm.AzureOpenAIConfig().get_supported_openai_params(model=model)
|
| 143 |
+
elif custom_llm_provider == "openrouter":
|
| 144 |
+
return litellm.OpenrouterConfig().get_supported_openai_params(model=model)
|
| 145 |
+
elif custom_llm_provider == "vercel_ai_gateway":
|
| 146 |
+
return litellm.VercelAIGatewayConfig().get_supported_openai_params(model=model)
|
| 147 |
+
elif custom_llm_provider == "mistral" or custom_llm_provider == "codestral":
|
| 148 |
+
# mistal and codestral api have the exact same params
|
| 149 |
+
if request_type == "chat_completion":
|
| 150 |
+
return litellm.MistralConfig().get_supported_openai_params(model=model)
|
| 151 |
+
elif request_type == "embeddings":
|
| 152 |
+
return litellm.MistralEmbeddingConfig().get_supported_openai_params()
|
| 153 |
+
elif request_type == "transcription":
|
| 154 |
+
from litellm.llms.mistral.audio_transcription.transformation import (
|
| 155 |
+
MistralAudioTranscriptionConfig,
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
return MistralAudioTranscriptionConfig().get_supported_openai_params(
|
| 159 |
+
model=model
|
| 160 |
+
)
|
| 161 |
+
elif custom_llm_provider == "text-completion-codestral":
|
| 162 |
+
return litellm.CodestralTextCompletionConfig().get_supported_openai_params(
|
| 163 |
+
model=model
|
| 164 |
+
)
|
| 165 |
+
elif custom_llm_provider == "sambanova":
|
| 166 |
+
if request_type == "embeddings":
|
| 167 |
+
litellm.SambaNovaEmbeddingConfig().get_supported_openai_params(model=model)
|
| 168 |
+
else:
|
| 169 |
+
return litellm.SambanovaConfig().get_supported_openai_params(model=model)
|
| 170 |
+
elif custom_llm_provider == "nebius":
|
| 171 |
+
if request_type == "chat_completion":
|
| 172 |
+
return litellm.NebiusConfig().get_supported_openai_params(model=model)
|
| 173 |
+
elif custom_llm_provider == "wandb":
|
| 174 |
+
if request_type == "chat_completion":
|
| 175 |
+
return litellm.WandbConfig().get_supported_openai_params(model=model)
|
| 176 |
+
elif custom_llm_provider == "replicate":
|
| 177 |
+
return litellm.ReplicateConfig().get_supported_openai_params(model=model)
|
| 178 |
+
elif custom_llm_provider == "huggingface":
|
| 179 |
+
return litellm.HuggingFaceChatConfig().get_supported_openai_params(model=model)
|
| 180 |
+
elif custom_llm_provider == "jina_ai":
|
| 181 |
+
if request_type == "embeddings":
|
| 182 |
+
return litellm.JinaAIEmbeddingConfig().get_supported_openai_params(
|
| 183 |
+
model=model
|
| 184 |
+
)
|
| 185 |
+
elif custom_llm_provider == "together_ai":
|
| 186 |
+
return litellm.TogetherAIConfig().get_supported_openai_params(model=model)
|
| 187 |
+
elif custom_llm_provider == "databricks":
|
| 188 |
+
if request_type == "chat_completion":
|
| 189 |
+
return litellm.DatabricksConfig().get_supported_openai_params(model=model)
|
| 190 |
+
elif request_type == "embeddings":
|
| 191 |
+
return litellm.DatabricksEmbeddingConfig().get_supported_openai_params()
|
| 192 |
+
elif custom_llm_provider == "palm" or custom_llm_provider == "gemini":
|
| 193 |
+
return litellm.GoogleAIStudioGeminiConfig().get_supported_openai_params(
|
| 194 |
+
model=model
|
| 195 |
+
)
|
| 196 |
+
elif custom_llm_provider == "novita":
|
| 197 |
+
return litellm.NovitaConfig().get_supported_openai_params(model=model)
|
| 198 |
+
elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta":
|
| 199 |
+
if request_type == "chat_completion":
|
| 200 |
+
if model.startswith("mistral"):
|
| 201 |
+
return litellm.MistralConfig().get_supported_openai_params(model=model)
|
| 202 |
+
elif model.startswith("codestral"):
|
| 203 |
+
return (
|
| 204 |
+
litellm.CodestralTextCompletionConfig().get_supported_openai_params(
|
| 205 |
+
model=model
|
| 206 |
+
)
|
| 207 |
+
)
|
| 208 |
+
elif model.startswith("claude"):
|
| 209 |
+
return litellm.VertexAIAnthropicConfig().get_supported_openai_params(
|
| 210 |
+
model=model
|
| 211 |
+
)
|
| 212 |
+
elif model.startswith("gemini"):
|
| 213 |
+
return litellm.VertexGeminiConfig().get_supported_openai_params(
|
| 214 |
+
model=model
|
| 215 |
+
)
|
| 216 |
+
else:
|
| 217 |
+
return litellm.VertexAILlama3Config().get_supported_openai_params(
|
| 218 |
+
model=model
|
| 219 |
+
)
|
| 220 |
+
elif request_type == "embeddings":
|
| 221 |
+
return litellm.VertexAITextEmbeddingConfig().get_supported_openai_params()
|
| 222 |
+
elif custom_llm_provider == "sagemaker":
|
| 223 |
+
return litellm.SagemakerConfig().get_supported_openai_params(model=model)
|
| 224 |
+
elif custom_llm_provider == "aleph_alpha":
|
| 225 |
+
return [
|
| 226 |
+
"max_tokens",
|
| 227 |
+
"stream",
|
| 228 |
+
"top_p",
|
| 229 |
+
"temperature",
|
| 230 |
+
"presence_penalty",
|
| 231 |
+
"frequency_penalty",
|
| 232 |
+
"n",
|
| 233 |
+
"stop",
|
| 234 |
+
]
|
| 235 |
+
elif custom_llm_provider == "cloudflare":
|
| 236 |
+
return litellm.CloudflareChatConfig().get_supported_openai_params(model=model)
|
| 237 |
+
elif custom_llm_provider == "nlp_cloud":
|
| 238 |
+
return litellm.NLPCloudConfig().get_supported_openai_params(model=model)
|
| 239 |
+
elif custom_llm_provider == "petals":
|
| 240 |
+
return litellm.PetalsConfig().get_supported_openai_params(model=model)
|
| 241 |
+
elif custom_llm_provider == "deepinfra":
|
| 242 |
+
return litellm.DeepInfraConfig().get_supported_openai_params(model=model)
|
| 243 |
+
elif custom_llm_provider == "perplexity":
|
| 244 |
+
return litellm.PerplexityChatConfig().get_supported_openai_params(model=model)
|
| 245 |
+
elif custom_llm_provider == "nscale":
|
| 246 |
+
return litellm.NscaleConfig().get_supported_openai_params(model=model)
|
| 247 |
+
elif custom_llm_provider == "anyscale":
|
| 248 |
+
return [
|
| 249 |
+
"temperature",
|
| 250 |
+
"top_p",
|
| 251 |
+
"stream",
|
| 252 |
+
"max_tokens",
|
| 253 |
+
"stop",
|
| 254 |
+
"frequency_penalty",
|
| 255 |
+
"presence_penalty",
|
| 256 |
+
]
|
| 257 |
+
elif custom_llm_provider == "watsonx":
|
| 258 |
+
return litellm.IBMWatsonXChatConfig().get_supported_openai_params(model=model)
|
| 259 |
+
elif custom_llm_provider == "watsonx_text":
|
| 260 |
+
return litellm.IBMWatsonXAIConfig().get_supported_openai_params(model=model)
|
| 261 |
+
elif (
|
| 262 |
+
custom_llm_provider == "custom_openai"
|
| 263 |
+
or custom_llm_provider == "text-completion-openai"
|
| 264 |
+
):
|
| 265 |
+
return litellm.OpenAITextCompletionConfig().get_supported_openai_params(
|
| 266 |
+
model=model
|
| 267 |
+
)
|
| 268 |
+
elif custom_llm_provider == "predibase":
|
| 269 |
+
return litellm.PredibaseConfig().get_supported_openai_params(model=model)
|
| 270 |
+
elif custom_llm_provider == "voyage":
|
| 271 |
+
return litellm.VoyageEmbeddingConfig().get_supported_openai_params(model=model)
|
| 272 |
+
elif custom_llm_provider == "infinity":
|
| 273 |
+
return litellm.InfinityEmbeddingConfig().get_supported_openai_params(
|
| 274 |
+
model=model
|
| 275 |
+
)
|
| 276 |
+
elif custom_llm_provider == "triton":
|
| 277 |
+
if request_type == "embeddings":
|
| 278 |
+
return litellm.TritonEmbeddingConfig().get_supported_openai_params(
|
| 279 |
+
model=model
|
| 280 |
+
)
|
| 281 |
+
else:
|
| 282 |
+
return litellm.TritonConfig().get_supported_openai_params(model=model)
|
| 283 |
+
elif custom_llm_provider == "deepgram":
|
| 284 |
+
if request_type == "transcription":
|
| 285 |
+
return (
|
| 286 |
+
litellm.DeepgramAudioTranscriptionConfig().get_supported_openai_params(
|
| 287 |
+
model=model
|
| 288 |
+
)
|
| 289 |
+
)
|
| 290 |
+
elif custom_llm_provider == "ovhcloud":
|
| 291 |
+
if request_type == "transcription":
|
| 292 |
+
from litellm.llms.ovhcloud.audio_transcription.transformation import (
|
| 293 |
+
OVHCloudAudioTranscriptionConfig,
|
| 294 |
+
)
|
| 295 |
+
|
| 296 |
+
return OVHCloudAudioTranscriptionConfig().get_supported_openai_params(
|
| 297 |
+
model=model
|
| 298 |
+
)
|
| 299 |
+
elif custom_llm_provider == "elevenlabs":
|
| 300 |
+
if request_type == "transcription":
|
| 301 |
+
from litellm.llms.elevenlabs.audio_transcription.transformation import (
|
| 302 |
+
ElevenLabsAudioTranscriptionConfig,
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
return ElevenLabsAudioTranscriptionConfig().get_supported_openai_params(
|
| 306 |
+
model=model
|
| 307 |
+
)
|
| 308 |
+
elif custom_llm_provider in litellm._custom_providers:
|
| 309 |
+
if request_type == "chat_completion":
|
| 310 |
+
provider_config = litellm.ProviderConfigManager.get_provider_chat_config(
|
| 311 |
+
model=model, provider=LlmProviders.CUSTOM
|
| 312 |
+
)
|
| 313 |
+
if provider_config:
|
| 314 |
+
return provider_config.get_supported_openai_params(model=model)
|
| 315 |
+
elif request_type == "embeddings":
|
| 316 |
+
return None
|
| 317 |
+
elif request_type == "transcription":
|
| 318 |
+
return None
|
| 319 |
+
|
| 320 |
+
return None
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/health_check_helpers.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Helper functions for health check calls.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from typing import TYPE_CHECKING, Callable, Dict, Literal, Optional
|
| 6 |
+
|
| 7 |
+
from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS
|
| 8 |
+
|
| 9 |
+
if TYPE_CHECKING:
|
| 10 |
+
from litellm.litellm_core_utils.litellm_logging import Logging
|
| 11 |
+
|
| 12 |
+
# Minimal PDF for health checks - base64 encoded 1-page PDF with just "test"
|
| 13 |
+
TEST_PDF_URL = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class HealthCheckHelpers:
|
| 17 |
+
@staticmethod
|
| 18 |
+
async def ahealth_check_wildcard_models(
|
| 19 |
+
model: str,
|
| 20 |
+
custom_llm_provider: str,
|
| 21 |
+
model_params: dict,
|
| 22 |
+
litellm_logging_obj: "Logging",
|
| 23 |
+
) -> dict:
|
| 24 |
+
from litellm import acompletion
|
| 25 |
+
from litellm.litellm_core_utils.llm_request_utils import (
|
| 26 |
+
pick_cheapest_chat_models_from_llm_provider,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
# this is a wildcard model, we need to pick a random model from the provider
|
| 30 |
+
cheapest_models = pick_cheapest_chat_models_from_llm_provider(
|
| 31 |
+
custom_llm_provider=custom_llm_provider, n=3
|
| 32 |
+
)
|
| 33 |
+
if len(cheapest_models) == 0:
|
| 34 |
+
raise Exception(
|
| 35 |
+
f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
|
| 36 |
+
)
|
| 37 |
+
if len(cheapest_models) > 1:
|
| 38 |
+
fallback_models = cheapest_models[
|
| 39 |
+
1:
|
| 40 |
+
] # Pick the last 2 models from the shuffled list
|
| 41 |
+
else:
|
| 42 |
+
fallback_models = None
|
| 43 |
+
model_params["model"] = cheapest_models[0]
|
| 44 |
+
model_params["litellm_logging_obj"] = litellm_logging_obj
|
| 45 |
+
model_params["fallbacks"] = fallback_models
|
| 46 |
+
model_params["max_tokens"] = model_params.get(
|
| 47 |
+
"max_tokens", 10
|
| 48 |
+
) # gpt-5-nano throws errors for max_tokens=1
|
| 49 |
+
await acompletion(**model_params)
|
| 50 |
+
return {}
|
| 51 |
+
|
| 52 |
+
@staticmethod
|
| 53 |
+
def _update_model_params_with_health_check_tracking_information(
|
| 54 |
+
model_params: dict,
|
| 55 |
+
) -> dict:
|
| 56 |
+
"""
|
| 57 |
+
Updates the health check model params with tracking information.
|
| 58 |
+
|
| 59 |
+
The following is added at this stage:
|
| 60 |
+
1. `tags`: This helps identify health check calls in the DB.
|
| 61 |
+
2. `user_api_key_auth`: This helps identify health check calls in the DB.
|
| 62 |
+
We need this since the DB requires an API Key to track a log in the SpendLogs Table
|
| 63 |
+
"""
|
| 64 |
+
from litellm.proxy._types import UserAPIKeyAuth
|
| 65 |
+
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
|
| 66 |
+
|
| 67 |
+
_metadata_variable_name = "litellm_metadata"
|
| 68 |
+
litellm_metadata = HealthCheckHelpers._get_metadata_for_health_check_call()
|
| 69 |
+
model_params[_metadata_variable_name] = litellm_metadata
|
| 70 |
+
model_params = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
|
| 71 |
+
data=model_params,
|
| 72 |
+
user_api_key_dict=UserAPIKeyAuth.get_litellm_internal_health_check_user_api_key_auth(),
|
| 73 |
+
_metadata_variable_name=_metadata_variable_name,
|
| 74 |
+
)
|
| 75 |
+
return model_params
|
| 76 |
+
|
| 77 |
+
@staticmethod
|
| 78 |
+
def _get_metadata_for_health_check_call():
|
| 79 |
+
"""
|
| 80 |
+
Returns the metadata for the health check call.
|
| 81 |
+
"""
|
| 82 |
+
from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME
|
| 83 |
+
|
| 84 |
+
return {
|
| 85 |
+
"tags": [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME],
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
@staticmethod
|
| 89 |
+
async def _batch_health_check(
|
| 90 |
+
custom_llm_provider: str,
|
| 91 |
+
model_params: dict,
|
| 92 |
+
filtered_model_params: dict,
|
| 93 |
+
) -> dict:
|
| 94 |
+
"""
|
| 95 |
+
Health check for batch mode.
|
| 96 |
+
|
| 97 |
+
Calls list_batches for providers that support it (openai, hosted_vllm, azure,
|
| 98 |
+
vertex_ai). For all other providers (e.g. bedrock) the batch API surface doesn't
|
| 99 |
+
include list_batches, so we fall back to acompletion to verify connectivity and
|
| 100 |
+
credential validity instead.
|
| 101 |
+
"""
|
| 102 |
+
import litellm
|
| 103 |
+
|
| 104 |
+
if custom_llm_provider in LIST_BATCHES_SUPPORTED_PROVIDERS:
|
| 105 |
+
return await litellm.alist_batches(**filtered_model_params)
|
| 106 |
+
else:
|
| 107 |
+
return await litellm.acompletion(**model_params)
|
| 108 |
+
|
| 109 |
+
@staticmethod
|
| 110 |
+
def get_mode_handlers(
|
| 111 |
+
model: str,
|
| 112 |
+
custom_llm_provider: str,
|
| 113 |
+
model_params: dict,
|
| 114 |
+
prompt: Optional[str] = None,
|
| 115 |
+
input: Optional[list] = None,
|
| 116 |
+
) -> Dict[
|
| 117 |
+
Literal[
|
| 118 |
+
"chat",
|
| 119 |
+
"completion",
|
| 120 |
+
"embedding",
|
| 121 |
+
"audio_speech",
|
| 122 |
+
"audio_transcription",
|
| 123 |
+
"image_generation",
|
| 124 |
+
"video_generation",
|
| 125 |
+
"rerank",
|
| 126 |
+
"realtime",
|
| 127 |
+
"batch",
|
| 128 |
+
"responses",
|
| 129 |
+
"ocr",
|
| 130 |
+
],
|
| 131 |
+
Callable,
|
| 132 |
+
]:
|
| 133 |
+
"""
|
| 134 |
+
Returns a dictionary of mode handlers for health check calls.
|
| 135 |
+
|
| 136 |
+
Mode Handlers are Callables that need to be run for execution of the health check call.
|
| 137 |
+
|
| 138 |
+
Args:
|
| 139 |
+
model: The model name
|
| 140 |
+
custom_llm_provider: The LLM provider
|
| 141 |
+
model_params: The model parameters
|
| 142 |
+
prompt: Optional prompt for health check
|
| 143 |
+
input: Optional input for health check
|
| 144 |
+
|
| 145 |
+
Returns:
|
| 146 |
+
Dictionary mapping mode names to their handler functions
|
| 147 |
+
"""
|
| 148 |
+
import litellm
|
| 149 |
+
from litellm.litellm_core_utils.audio_utils.utils import (
|
| 150 |
+
get_audio_file_for_health_check,
|
| 151 |
+
)
|
| 152 |
+
from litellm.litellm_core_utils.health_check_utils import _filter_model_params
|
| 153 |
+
from litellm.realtime_api.main import _realtime_health_check
|
| 154 |
+
|
| 155 |
+
return {
|
| 156 |
+
"chat": lambda: litellm.acompletion(
|
| 157 |
+
**model_params,
|
| 158 |
+
),
|
| 159 |
+
"completion": lambda: litellm.atext_completion(
|
| 160 |
+
**_filter_model_params(model_params=model_params),
|
| 161 |
+
prompt=prompt or "test",
|
| 162 |
+
),
|
| 163 |
+
"embedding": lambda: litellm.aembedding(
|
| 164 |
+
**_filter_model_params(model_params=model_params),
|
| 165 |
+
input=input or ["test"],
|
| 166 |
+
),
|
| 167 |
+
"audio_speech": lambda: litellm.aspeech(
|
| 168 |
+
**{
|
| 169 |
+
**_filter_model_params(model_params=model_params),
|
| 170 |
+
**(
|
| 171 |
+
{"voice": "alloy"}
|
| 172 |
+
if "voice"
|
| 173 |
+
not in _filter_model_params(model_params=model_params)
|
| 174 |
+
else {}
|
| 175 |
+
),
|
| 176 |
+
},
|
| 177 |
+
input=prompt or "test",
|
| 178 |
+
),
|
| 179 |
+
"audio_transcription": lambda: litellm.atranscription(
|
| 180 |
+
**_filter_model_params(model_params=model_params),
|
| 181 |
+
file=get_audio_file_for_health_check(),
|
| 182 |
+
),
|
| 183 |
+
"image_generation": lambda: litellm.aimage_generation(
|
| 184 |
+
**_filter_model_params(model_params=model_params),
|
| 185 |
+
prompt=prompt,
|
| 186 |
+
),
|
| 187 |
+
"video_generation": lambda: litellm.avideo_generation(
|
| 188 |
+
**_filter_model_params(model_params=model_params),
|
| 189 |
+
prompt=prompt or "test video generation",
|
| 190 |
+
),
|
| 191 |
+
"rerank": lambda: litellm.arerank(
|
| 192 |
+
**_filter_model_params(model_params=model_params),
|
| 193 |
+
query=prompt or "",
|
| 194 |
+
documents=["my sample text"],
|
| 195 |
+
),
|
| 196 |
+
"realtime": lambda: _realtime_health_check(
|
| 197 |
+
model=model,
|
| 198 |
+
custom_llm_provider=custom_llm_provider,
|
| 199 |
+
api_base=model_params.get("api_base", None),
|
| 200 |
+
api_key=model_params.get("api_key", None),
|
| 201 |
+
api_version=model_params.get("api_version", None),
|
| 202 |
+
),
|
| 203 |
+
"batch": lambda: HealthCheckHelpers._batch_health_check(
|
| 204 |
+
custom_llm_provider=custom_llm_provider,
|
| 205 |
+
model_params=model_params,
|
| 206 |
+
filtered_model_params=_filter_model_params(model_params=model_params),
|
| 207 |
+
),
|
| 208 |
+
"responses": lambda: litellm.aresponses(
|
| 209 |
+
**_filter_model_params(model_params=model_params),
|
| 210 |
+
input=prompt or "test",
|
| 211 |
+
),
|
| 212 |
+
"ocr": lambda: litellm.aocr(
|
| 213 |
+
**_filter_model_params(model_params=model_params),
|
| 214 |
+
document={
|
| 215 |
+
"type": "document_url",
|
| 216 |
+
"document_url": TEST_PDF_URL,
|
| 217 |
+
},
|
| 218 |
+
),
|
| 219 |
+
}
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/health_check_utils.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Utils used for litellm.ahealth_check()
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def _filter_model_params(model_params: dict) -> dict:
|
| 7 |
+
"""Remove 'messages' param from model params."""
|
| 8 |
+
return {k: v for k, v in model_params.items() if k != "messages"}
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _create_health_check_response(response_headers: dict) -> dict:
|
| 12 |
+
response = {}
|
| 13 |
+
|
| 14 |
+
if (
|
| 15 |
+
response_headers.get("x-ratelimit-remaining-requests", None) is not None
|
| 16 |
+
): # not provided for dall-e requests
|
| 17 |
+
response["x-ratelimit-remaining-requests"] = response_headers[
|
| 18 |
+
"x-ratelimit-remaining-requests"
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
if response_headers.get("x-ratelimit-remaining-tokens", None) is not None:
|
| 22 |
+
response["x-ratelimit-remaining-tokens"] = response_headers[
|
| 23 |
+
"x-ratelimit-remaining-tokens"
|
| 24 |
+
]
|
| 25 |
+
|
| 26 |
+
if response_headers.get("x-ms-region", None) is not None:
|
| 27 |
+
response["x-ms-region"] = response_headers["x-ms-region"]
|
| 28 |
+
return response
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/initialize_dynamic_callback_params.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, Optional
|
| 2 |
+
from litellm.secret_managers.main import get_secret_str
|
| 3 |
+
from litellm.types.utils import StandardCallbackDynamicParams
|
| 4 |
+
|
| 5 |
+
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
|
| 6 |
+
_supported_callback_params = [
|
| 7 |
+
"langfuse_public_key",
|
| 8 |
+
"langfuse_secret",
|
| 9 |
+
"langfuse_secret_key",
|
| 10 |
+
"langfuse_host",
|
| 11 |
+
"langfuse_prompt_version",
|
| 12 |
+
"gcs_bucket_name",
|
| 13 |
+
"gcs_path_service_account",
|
| 14 |
+
"langsmith_api_key",
|
| 15 |
+
"langsmith_project",
|
| 16 |
+
"langsmith_base_url",
|
| 17 |
+
"langsmith_sampling_rate",
|
| 18 |
+
"langsmith_tenant_id",
|
| 19 |
+
"humanloop_api_key",
|
| 20 |
+
"arize_api_key",
|
| 21 |
+
"arize_space_key",
|
| 22 |
+
"arize_space_id",
|
| 23 |
+
"posthog_api_key",
|
| 24 |
+
"posthog_host",
|
| 25 |
+
"braintrust_api_key",
|
| 26 |
+
"braintrust_project",
|
| 27 |
+
"braintrust_host",
|
| 28 |
+
"slack_webhook_url",
|
| 29 |
+
"lunary_public_key",
|
| 30 |
+
"turn_off_message_logging",
|
| 31 |
+
]
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def initialize_standard_callback_dynamic_params(
|
| 35 |
+
kwargs: Optional[Dict] = None,
|
| 36 |
+
) -> StandardCallbackDynamicParams:
|
| 37 |
+
"""
|
| 38 |
+
Initialize the standard callback dynamic params from the kwargs
|
| 39 |
+
|
| 40 |
+
checks if langfuse_secret_key, gcs_bucket_name in kwargs and sets the corresponding attributes in StandardCallbackDynamicParams
|
| 41 |
+
"""
|
| 42 |
+
|
| 43 |
+
standard_callback_dynamic_params = StandardCallbackDynamicParams()
|
| 44 |
+
if kwargs:
|
| 45 |
+
# 1. Check top-level kwargs
|
| 46 |
+
for param in _supported_callback_params:
|
| 47 |
+
if param in kwargs:
|
| 48 |
+
_param_value = kwargs.get(param)
|
| 49 |
+
if (
|
| 50 |
+
_param_value is not None
|
| 51 |
+
and isinstance(_param_value, str)
|
| 52 |
+
and "os.environ/" in _param_value
|
| 53 |
+
):
|
| 54 |
+
_param_value = get_secret_str(secret_name=_param_value)
|
| 55 |
+
standard_callback_dynamic_params[param] = _param_value # type: ignore
|
| 56 |
+
|
| 57 |
+
# 2. Fallback: check "metadata" or "litellm_params" -> "metadata"
|
| 58 |
+
metadata = (kwargs.get("metadata") or {}).copy()
|
| 59 |
+
litellm_params = kwargs.get("litellm_params") or {}
|
| 60 |
+
if isinstance(litellm_params, dict):
|
| 61 |
+
metadata.update(litellm_params.get("metadata") or {})
|
| 62 |
+
|
| 63 |
+
if isinstance(metadata, dict):
|
| 64 |
+
for param in _supported_callback_params:
|
| 65 |
+
if param not in standard_callback_dynamic_params and param in metadata:
|
| 66 |
+
_param_value = metadata.get(param)
|
| 67 |
+
if (
|
| 68 |
+
_param_value is not None
|
| 69 |
+
and isinstance(_param_value, str)
|
| 70 |
+
and "os.environ/" in _param_value
|
| 71 |
+
):
|
| 72 |
+
_param_value = get_secret_str(secret_name=_param_value)
|
| 73 |
+
standard_callback_dynamic_params[param] = _param_value # type: ignore
|
| 74 |
+
|
| 75 |
+
return standard_callback_dynamic_params
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/json_validation_rule.py
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from typing import Any, Dict, List, Union
|
| 3 |
+
|
| 4 |
+
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def normalize_json_schema_types(
|
| 8 |
+
schema: Union[Dict[str, Any], List[Any], Any],
|
| 9 |
+
depth: int = 0,
|
| 10 |
+
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
|
| 11 |
+
) -> Union[Dict[str, Any], List[Any], Any]:
|
| 12 |
+
"""
|
| 13 |
+
Normalize JSON schema types from uppercase to lowercase format.
|
| 14 |
+
|
| 15 |
+
Some providers (like certain Google services) use uppercase types like 'BOOLEAN', 'STRING', 'ARRAY', 'OBJECT'
|
| 16 |
+
but standard JSON Schema requires lowercase: 'boolean', 'string', 'array', 'object'
|
| 17 |
+
|
| 18 |
+
This function recursively normalizes all type fields in a schema to lowercase.
|
| 19 |
+
|
| 20 |
+
Args:
|
| 21 |
+
schema: The schema to normalize (dict, list, or other)
|
| 22 |
+
depth: Current recursion depth
|
| 23 |
+
max_depth: Maximum recursion depth to prevent infinite loops
|
| 24 |
+
|
| 25 |
+
Returns:
|
| 26 |
+
The normalized schema with lowercase types
|
| 27 |
+
"""
|
| 28 |
+
# Prevent infinite recursion
|
| 29 |
+
if depth >= max_depth:
|
| 30 |
+
return schema
|
| 31 |
+
|
| 32 |
+
if not isinstance(schema, (dict, list)):
|
| 33 |
+
return schema
|
| 34 |
+
|
| 35 |
+
# Type mapping from uppercase to lowercase
|
| 36 |
+
type_mapping = {
|
| 37 |
+
"BOOLEAN": "boolean",
|
| 38 |
+
"STRING": "string",
|
| 39 |
+
"ARRAY": "array",
|
| 40 |
+
"OBJECT": "object",
|
| 41 |
+
"NUMBER": "number",
|
| 42 |
+
"INTEGER": "integer",
|
| 43 |
+
"NULL": "null",
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
if isinstance(schema, list):
|
| 47 |
+
return [
|
| 48 |
+
normalize_json_schema_types(item, depth + 1, max_depth) for item in schema
|
| 49 |
+
]
|
| 50 |
+
|
| 51 |
+
if isinstance(schema, dict):
|
| 52 |
+
normalized_schema: Dict[str, Any] = {}
|
| 53 |
+
|
| 54 |
+
for key, value in schema.items():
|
| 55 |
+
if key == "type" and isinstance(value, str) and value in type_mapping:
|
| 56 |
+
normalized_schema[key] = type_mapping[value]
|
| 57 |
+
elif key == "properties" and isinstance(value, dict):
|
| 58 |
+
# Recursively normalize properties
|
| 59 |
+
normalized_schema[key] = {
|
| 60 |
+
prop_key: normalize_json_schema_types(
|
| 61 |
+
prop_value, depth + 1, max_depth
|
| 62 |
+
)
|
| 63 |
+
for prop_key, prop_value in value.items()
|
| 64 |
+
}
|
| 65 |
+
elif key == "items" and isinstance(value, (dict, list)):
|
| 66 |
+
# Recursively normalize array items
|
| 67 |
+
normalized_schema[key] = normalize_json_schema_types(
|
| 68 |
+
value, depth + 1, max_depth
|
| 69 |
+
)
|
| 70 |
+
elif isinstance(value, (dict, list)):
|
| 71 |
+
# Recursively normalize any nested dict or list
|
| 72 |
+
normalized_schema[key] = normalize_json_schema_types(
|
| 73 |
+
value, depth + 1, max_depth
|
| 74 |
+
)
|
| 75 |
+
else:
|
| 76 |
+
normalized_schema[key] = value
|
| 77 |
+
|
| 78 |
+
return normalized_schema
|
| 79 |
+
|
| 80 |
+
return schema
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def normalize_tool_schema(tool: Dict[str, Any]) -> Dict[str, Any]:
|
| 84 |
+
"""
|
| 85 |
+
Normalize a tool's parameter schema to use standard JSON Schema lowercase types.
|
| 86 |
+
|
| 87 |
+
Args:
|
| 88 |
+
tool: The tool definition containing function parameters
|
| 89 |
+
|
| 90 |
+
Returns:
|
| 91 |
+
The tool with normalized schema types
|
| 92 |
+
"""
|
| 93 |
+
if not isinstance(tool, dict):
|
| 94 |
+
return tool
|
| 95 |
+
|
| 96 |
+
normalized_tool = tool.copy()
|
| 97 |
+
|
| 98 |
+
# Normalize function parameters if present
|
| 99 |
+
if "function" in tool and isinstance(tool["function"], dict):
|
| 100 |
+
normalized_tool["function"] = tool["function"].copy()
|
| 101 |
+
if "parameters" in tool["function"]:
|
| 102 |
+
normalized_tool["function"]["parameters"] = normalize_json_schema_types(
|
| 103 |
+
tool["function"]["parameters"]
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
return normalized_tool
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def validate_schema(schema: dict, response: str):
|
| 110 |
+
"""
|
| 111 |
+
Validate if the returned json response follows the schema.
|
| 112 |
+
|
| 113 |
+
Params:
|
| 114 |
+
- schema - dict: JSON schema
|
| 115 |
+
- response - str: Received json response as string.
|
| 116 |
+
"""
|
| 117 |
+
from jsonschema import ValidationError, validate
|
| 118 |
+
|
| 119 |
+
from litellm import JSONSchemaValidationError
|
| 120 |
+
|
| 121 |
+
try:
|
| 122 |
+
response_dict = json.loads(response)
|
| 123 |
+
except json.JSONDecodeError:
|
| 124 |
+
raise JSONSchemaValidationError(
|
| 125 |
+
model="", llm_provider="", raw_response=response, schema=json.dumps(schema)
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
try:
|
| 129 |
+
validate(response_dict, schema=schema)
|
| 130 |
+
except ValidationError:
|
| 131 |
+
raise JSONSchemaValidationError(
|
| 132 |
+
model="", llm_provider="", raw_response=response, schema=json.dumps(schema)
|
| 133 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/litellm_logging.py
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
ADDED
|
@@ -0,0 +1,729 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Helper utilities for tracking the cost of built-in tools.
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from typing import Any, Dict, List, Literal, Optional, Tuple
|
| 6 |
+
|
| 7 |
+
import litellm
|
| 8 |
+
from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
|
| 9 |
+
from litellm.types.llms.openai import (
|
| 10 |
+
FileSearchTool,
|
| 11 |
+
ResponsesAPIResponse,
|
| 12 |
+
WebSearchOptions,
|
| 13 |
+
)
|
| 14 |
+
from litellm.types.utils import (
|
| 15 |
+
Message,
|
| 16 |
+
ModelInfo,
|
| 17 |
+
ModelResponse,
|
| 18 |
+
SearchContextCostPerQuery,
|
| 19 |
+
StandardBuiltInToolsParams,
|
| 20 |
+
Usage,
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class StandardBuiltInToolCostTracking:
|
| 25 |
+
"""
|
| 26 |
+
Helper class for tracking the cost of built-in tools
|
| 27 |
+
|
| 28 |
+
Example: Web Search
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
@staticmethod
|
| 32 |
+
def get_cost_for_built_in_tools(
|
| 33 |
+
model: str,
|
| 34 |
+
response_object: Any,
|
| 35 |
+
usage: Optional[Usage] = None,
|
| 36 |
+
custom_llm_provider: Optional[str] = None,
|
| 37 |
+
standard_built_in_tools_params: Optional[StandardBuiltInToolsParams] = None,
|
| 38 |
+
) -> float:
|
| 39 |
+
"""
|
| 40 |
+
Get the cost of using built-in tools.
|
| 41 |
+
|
| 42 |
+
Supported tools:
|
| 43 |
+
- Web Search
|
| 44 |
+
- File Search
|
| 45 |
+
- Vector Store (Azure)
|
| 46 |
+
- Computer Use (Azure)
|
| 47 |
+
- Code Interpreter (Azure)
|
| 48 |
+
"""
|
| 49 |
+
standard_built_in_tools_params = standard_built_in_tools_params or {}
|
| 50 |
+
|
| 51 |
+
# Handle web search
|
| 52 |
+
if StandardBuiltInToolCostTracking.response_object_includes_web_search_call(
|
| 53 |
+
response_object=response_object, usage=usage
|
| 54 |
+
):
|
| 55 |
+
return StandardBuiltInToolCostTracking._handle_web_search_cost(
|
| 56 |
+
model=model,
|
| 57 |
+
custom_llm_provider=custom_llm_provider,
|
| 58 |
+
usage=usage,
|
| 59 |
+
standard_built_in_tools_params=standard_built_in_tools_params,
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
# Handle file search
|
| 63 |
+
if StandardBuiltInToolCostTracking.response_object_includes_file_search_call(
|
| 64 |
+
response_object=response_object
|
| 65 |
+
):
|
| 66 |
+
return StandardBuiltInToolCostTracking._handle_file_search_cost(
|
| 67 |
+
model=model,
|
| 68 |
+
custom_llm_provider=custom_llm_provider,
|
| 69 |
+
standard_built_in_tools_params=standard_built_in_tools_params,
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
# Handle Azure assistant features
|
| 73 |
+
return StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
|
| 74 |
+
model=model,
|
| 75 |
+
custom_llm_provider=custom_llm_provider,
|
| 76 |
+
standard_built_in_tools_params=standard_built_in_tools_params,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
@staticmethod
|
| 80 |
+
def _handle_web_search_cost(
|
| 81 |
+
model: str,
|
| 82 |
+
custom_llm_provider: Optional[str],
|
| 83 |
+
usage: Optional[Usage],
|
| 84 |
+
standard_built_in_tools_params: StandardBuiltInToolsParams,
|
| 85 |
+
) -> float:
|
| 86 |
+
"""Handle web search cost calculation."""
|
| 87 |
+
from litellm.llms import get_cost_for_web_search_request
|
| 88 |
+
|
| 89 |
+
model_info = StandardBuiltInToolCostTracking._safe_get_model_info(
|
| 90 |
+
model=model, custom_llm_provider=custom_llm_provider
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
if custom_llm_provider is None and model_info is not None:
|
| 94 |
+
custom_llm_provider = model_info["litellm_provider"]
|
| 95 |
+
|
| 96 |
+
if (
|
| 97 |
+
model_info is not None
|
| 98 |
+
and usage is not None
|
| 99 |
+
and custom_llm_provider is not None
|
| 100 |
+
):
|
| 101 |
+
result = get_cost_for_web_search_request(
|
| 102 |
+
custom_llm_provider=custom_llm_provider,
|
| 103 |
+
usage=usage,
|
| 104 |
+
model_info=model_info,
|
| 105 |
+
)
|
| 106 |
+
if result is not None:
|
| 107 |
+
return result
|
| 108 |
+
|
| 109 |
+
return StandardBuiltInToolCostTracking.get_cost_for_web_search(
|
| 110 |
+
web_search_options=standard_built_in_tools_params.get(
|
| 111 |
+
"web_search_options", None
|
| 112 |
+
),
|
| 113 |
+
model_info=model_info,
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
@staticmethod
|
| 117 |
+
def _handle_file_search_cost(
|
| 118 |
+
model: str,
|
| 119 |
+
custom_llm_provider: Optional[str],
|
| 120 |
+
standard_built_in_tools_params: StandardBuiltInToolsParams,
|
| 121 |
+
) -> float:
|
| 122 |
+
"""Handle file search cost calculation."""
|
| 123 |
+
model_info = StandardBuiltInToolCostTracking._safe_get_model_info(
|
| 124 |
+
model=model, custom_llm_provider=custom_llm_provider
|
| 125 |
+
)
|
| 126 |
+
file_search_raw: Any = standard_built_in_tools_params.get("file_search", {})
|
| 127 |
+
file_search_usage: Optional[FileSearchTool] = (
|
| 128 |
+
FileSearchTool(**file_search_raw) if file_search_raw else None
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
# Convert model_info to dict and extract usage parameters
|
| 132 |
+
model_info_dict = dict(model_info) if model_info is not None else None
|
| 133 |
+
storage_gb, days = StandardBuiltInToolCostTracking._extract_file_search_params(
|
| 134 |
+
file_search_usage
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
return StandardBuiltInToolCostTracking.get_cost_for_file_search(
|
| 138 |
+
file_search=file_search_usage,
|
| 139 |
+
provider=custom_llm_provider,
|
| 140 |
+
model_info=model_info_dict,
|
| 141 |
+
storage_gb=storage_gb,
|
| 142 |
+
days=days,
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
@staticmethod
|
| 146 |
+
def _handle_azure_assistant_costs(
|
| 147 |
+
model: str,
|
| 148 |
+
custom_llm_provider: Optional[str],
|
| 149 |
+
standard_built_in_tools_params: StandardBuiltInToolsParams,
|
| 150 |
+
) -> float:
|
| 151 |
+
"""Handle Azure assistant features cost calculation."""
|
| 152 |
+
if custom_llm_provider != "azure":
|
| 153 |
+
return 0.0
|
| 154 |
+
|
| 155 |
+
model_info = StandardBuiltInToolCostTracking._safe_get_model_info(
|
| 156 |
+
model=model, custom_llm_provider=custom_llm_provider
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
total_cost = 0.0
|
| 160 |
+
total_cost += StandardBuiltInToolCostTracking._get_vector_store_cost(
|
| 161 |
+
model_info, custom_llm_provider, standard_built_in_tools_params
|
| 162 |
+
)
|
| 163 |
+
total_cost += StandardBuiltInToolCostTracking._get_computer_use_cost(
|
| 164 |
+
model_info, custom_llm_provider, standard_built_in_tools_params
|
| 165 |
+
)
|
| 166 |
+
total_cost += StandardBuiltInToolCostTracking._get_code_interpreter_cost(
|
| 167 |
+
model_info, custom_llm_provider, standard_built_in_tools_params
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
return total_cost
|
| 171 |
+
|
| 172 |
+
@staticmethod
|
| 173 |
+
def _extract_file_search_params(
|
| 174 |
+
file_search_usage: Any,
|
| 175 |
+
) -> Tuple[Optional[float], Optional[float]]:
|
| 176 |
+
"""Extract and convert file search parameters safely."""
|
| 177 |
+
storage_gb = None
|
| 178 |
+
days = None
|
| 179 |
+
|
| 180 |
+
if isinstance(file_search_usage, dict):
|
| 181 |
+
storage_gb_val = file_search_usage.get("storage_gb")
|
| 182 |
+
days_val = file_search_usage.get("days")
|
| 183 |
+
|
| 184 |
+
if storage_gb_val is not None:
|
| 185 |
+
try:
|
| 186 |
+
storage_gb = float(storage_gb_val) # type: ignore
|
| 187 |
+
except (TypeError, ValueError):
|
| 188 |
+
storage_gb = None
|
| 189 |
+
|
| 190 |
+
if days_val is not None:
|
| 191 |
+
try:
|
| 192 |
+
days = float(days_val) # type: ignore
|
| 193 |
+
except (TypeError, ValueError):
|
| 194 |
+
days = None
|
| 195 |
+
|
| 196 |
+
return storage_gb, days
|
| 197 |
+
|
| 198 |
+
@staticmethod
|
| 199 |
+
def _get_vector_store_cost(
|
| 200 |
+
model_info: Optional[ModelInfo],
|
| 201 |
+
custom_llm_provider: Optional[str],
|
| 202 |
+
standard_built_in_tools_params: StandardBuiltInToolsParams,
|
| 203 |
+
) -> float:
|
| 204 |
+
"""Calculate vector store cost."""
|
| 205 |
+
vector_store_usage = standard_built_in_tools_params.get(
|
| 206 |
+
"vector_store_usage", None
|
| 207 |
+
)
|
| 208 |
+
if not vector_store_usage:
|
| 209 |
+
return 0.0
|
| 210 |
+
|
| 211 |
+
model_info_dict = dict(model_info) if model_info is not None else None
|
| 212 |
+
vector_store_dict = (
|
| 213 |
+
vector_store_usage if isinstance(vector_store_usage, dict) else {}
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
return StandardBuiltInToolCostTracking.get_cost_for_vector_store(
|
| 217 |
+
vector_store_usage=vector_store_dict,
|
| 218 |
+
provider=custom_llm_provider,
|
| 219 |
+
model_info=model_info_dict,
|
| 220 |
+
)
|
| 221 |
+
|
| 222 |
+
@staticmethod
|
| 223 |
+
def _get_computer_use_cost(
|
| 224 |
+
model_info: Optional[ModelInfo],
|
| 225 |
+
custom_llm_provider: Optional[str],
|
| 226 |
+
standard_built_in_tools_params: StandardBuiltInToolsParams,
|
| 227 |
+
) -> float:
|
| 228 |
+
"""Calculate computer use cost."""
|
| 229 |
+
computer_use_usage = standard_built_in_tools_params.get(
|
| 230 |
+
"computer_use_usage", {}
|
| 231 |
+
)
|
| 232 |
+
if not computer_use_usage:
|
| 233 |
+
return 0.0
|
| 234 |
+
|
| 235 |
+
model_info_dict = dict(model_info) if model_info is not None else None
|
| 236 |
+
(
|
| 237 |
+
input_tokens,
|
| 238 |
+
output_tokens,
|
| 239 |
+
) = StandardBuiltInToolCostTracking._extract_token_counts(computer_use_usage)
|
| 240 |
+
|
| 241 |
+
return StandardBuiltInToolCostTracking.get_cost_for_computer_use(
|
| 242 |
+
input_tokens=input_tokens,
|
| 243 |
+
output_tokens=output_tokens,
|
| 244 |
+
provider=custom_llm_provider,
|
| 245 |
+
model_info=model_info_dict,
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
@staticmethod
|
| 249 |
+
def _get_code_interpreter_cost(
|
| 250 |
+
model_info: Optional[ModelInfo],
|
| 251 |
+
custom_llm_provider: Optional[str],
|
| 252 |
+
standard_built_in_tools_params: StandardBuiltInToolsParams,
|
| 253 |
+
) -> float:
|
| 254 |
+
"""Calculate code interpreter cost."""
|
| 255 |
+
code_interpreter_sessions = standard_built_in_tools_params.get(
|
| 256 |
+
"code_interpreter_sessions", None
|
| 257 |
+
)
|
| 258 |
+
if not code_interpreter_sessions:
|
| 259 |
+
return 0.0
|
| 260 |
+
|
| 261 |
+
model_info_dict = dict(model_info) if model_info is not None else None
|
| 262 |
+
sessions = StandardBuiltInToolCostTracking._safe_convert_to_int(
|
| 263 |
+
code_interpreter_sessions
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
return StandardBuiltInToolCostTracking.get_cost_for_code_interpreter(
|
| 267 |
+
sessions=sessions,
|
| 268 |
+
provider=custom_llm_provider,
|
| 269 |
+
model_info=model_info_dict,
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
@staticmethod
|
| 273 |
+
def _extract_token_counts(
|
| 274 |
+
computer_use_usage: Any,
|
| 275 |
+
) -> Tuple[Optional[int], Optional[int]]:
|
| 276 |
+
"""Extract and convert token counts safely."""
|
| 277 |
+
input_tokens = None
|
| 278 |
+
output_tokens = None
|
| 279 |
+
|
| 280 |
+
if isinstance(computer_use_usage, dict):
|
| 281 |
+
input_tokens_val = computer_use_usage.get("input_tokens")
|
| 282 |
+
output_tokens_val = computer_use_usage.get("output_tokens")
|
| 283 |
+
|
| 284 |
+
input_tokens = StandardBuiltInToolCostTracking._safe_convert_to_int(
|
| 285 |
+
input_tokens_val
|
| 286 |
+
)
|
| 287 |
+
output_tokens = StandardBuiltInToolCostTracking._safe_convert_to_int(
|
| 288 |
+
output_tokens_val
|
| 289 |
+
)
|
| 290 |
+
|
| 291 |
+
return input_tokens, output_tokens
|
| 292 |
+
|
| 293 |
+
@staticmethod
|
| 294 |
+
def _safe_convert_to_int(value: Any) -> Optional[int]:
|
| 295 |
+
"""Safely convert a value to int."""
|
| 296 |
+
if value is not None:
|
| 297 |
+
try:
|
| 298 |
+
return int(value) # type: ignore
|
| 299 |
+
except (TypeError, ValueError):
|
| 300 |
+
return None
|
| 301 |
+
return None
|
| 302 |
+
|
| 303 |
+
@staticmethod
|
| 304 |
+
def response_object_includes_web_search_call(
|
| 305 |
+
response_object: Any, usage: Optional[Usage] = None
|
| 306 |
+
) -> bool:
|
| 307 |
+
"""
|
| 308 |
+
Check if the response object includes a web search call.
|
| 309 |
+
|
| 310 |
+
This covers:
|
| 311 |
+
- Chat Completion Response (ModelResponse)
|
| 312 |
+
- ResponsesAPIResponse (streaming + non-streaming)
|
| 313 |
+
"""
|
| 314 |
+
from litellm.types.utils import PromptTokensDetailsWrapper
|
| 315 |
+
|
| 316 |
+
if isinstance(response_object, ModelResponse):
|
| 317 |
+
# chat completions only include url_citation annotations when a web search call is made
|
| 318 |
+
has_url_citations = (
|
| 319 |
+
StandardBuiltInToolCostTracking.response_includes_annotation_type(
|
| 320 |
+
response_object=response_object, annotation_type="url_citation"
|
| 321 |
+
)
|
| 322 |
+
)
|
| 323 |
+
if has_url_citations:
|
| 324 |
+
return True
|
| 325 |
+
# Fallback: Check usage object for providers that use usage instead of annotations
|
| 326 |
+
# (e.g., Vertex AI Gemini uses usage.prompt_tokens_details.web_search_requests)
|
| 327 |
+
if usage is not None:
|
| 328 |
+
if (
|
| 329 |
+
hasattr(usage, "prompt_tokens_details")
|
| 330 |
+
and usage.prompt_tokens_details is not None
|
| 331 |
+
and isinstance(
|
| 332 |
+
usage.prompt_tokens_details, PromptTokensDetailsWrapper
|
| 333 |
+
)
|
| 334 |
+
and hasattr(usage.prompt_tokens_details, "web_search_requests")
|
| 335 |
+
and usage.prompt_tokens_details.web_search_requests is not None
|
| 336 |
+
):
|
| 337 |
+
return True
|
| 338 |
+
return False
|
| 339 |
+
elif isinstance(response_object, ResponsesAPIResponse):
|
| 340 |
+
# response api explicitly includes web_search_call in the output
|
| 341 |
+
return StandardBuiltInToolCostTracking.response_includes_output_type(
|
| 342 |
+
response_object=response_object, output_type="web_search_call"
|
| 343 |
+
)
|
| 344 |
+
elif usage is not None:
|
| 345 |
+
if (
|
| 346 |
+
hasattr(usage, "server_tool_use")
|
| 347 |
+
and usage.server_tool_use is not None
|
| 348 |
+
and usage.server_tool_use.web_search_requests is not None
|
| 349 |
+
):
|
| 350 |
+
return True
|
| 351 |
+
elif (
|
| 352 |
+
hasattr(usage, "prompt_tokens_details")
|
| 353 |
+
and usage.prompt_tokens_details is not None
|
| 354 |
+
and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
|
| 355 |
+
and hasattr(usage.prompt_tokens_details, "web_search_requests")
|
| 356 |
+
and usage.prompt_tokens_details.web_search_requests is not None
|
| 357 |
+
):
|
| 358 |
+
return True
|
| 359 |
+
|
| 360 |
+
return False
|
| 361 |
+
|
| 362 |
+
@staticmethod
|
| 363 |
+
def response_object_includes_file_search_call(
|
| 364 |
+
response_object: Any,
|
| 365 |
+
) -> bool:
|
| 366 |
+
"""
|
| 367 |
+
Check if the response object includes a file search call.
|
| 368 |
+
|
| 369 |
+
This covers:
|
| 370 |
+
- Chat Completion Response (ModelResponse)
|
| 371 |
+
- ResponsesAPIResponse (streaming + non-streaming)
|
| 372 |
+
"""
|
| 373 |
+
if isinstance(response_object, ModelResponse):
|
| 374 |
+
# chat completions only include file_citation annotations when a file search call is made
|
| 375 |
+
return StandardBuiltInToolCostTracking.response_includes_annotation_type(
|
| 376 |
+
response_object=response_object, annotation_type="file_citation"
|
| 377 |
+
)
|
| 378 |
+
elif isinstance(response_object, ResponsesAPIResponse):
|
| 379 |
+
# response api explicitly includes file_search_call in the output
|
| 380 |
+
return StandardBuiltInToolCostTracking.response_includes_output_type(
|
| 381 |
+
response_object=response_object, output_type="file_search_call"
|
| 382 |
+
)
|
| 383 |
+
return False
|
| 384 |
+
|
| 385 |
+
@staticmethod
|
| 386 |
+
def response_includes_annotation_type(
|
| 387 |
+
response_object: ModelResponse,
|
| 388 |
+
annotation_type: Literal["url_citation", "file_citation"],
|
| 389 |
+
) -> bool:
|
| 390 |
+
if isinstance(response_object, ModelResponse):
|
| 391 |
+
for choice in response_object.choices:
|
| 392 |
+
message: Optional[Message] = getattr(choice, "message", None)
|
| 393 |
+
if message is None:
|
| 394 |
+
continue
|
| 395 |
+
if annotations := getattr(message, "annotations", None):
|
| 396 |
+
if len(annotations) > 0:
|
| 397 |
+
for annotation in annotations:
|
| 398 |
+
if annotation.get("type", None) == annotation_type:
|
| 399 |
+
return True
|
| 400 |
+
return False
|
| 401 |
+
|
| 402 |
+
@staticmethod
|
| 403 |
+
def response_includes_output_type(
|
| 404 |
+
response_object: ResponsesAPIResponse,
|
| 405 |
+
output_type: Literal["web_search_call", "file_search_call"],
|
| 406 |
+
) -> bool:
|
| 407 |
+
"""
|
| 408 |
+
Check if the ResponsesAPIResponse includes one of the specified output types.
|
| 409 |
+
|
| 410 |
+
This is used for cost tracking of built-in tools.
|
| 411 |
+
|
| 412 |
+
Args:
|
| 413 |
+
response_object: The ResponsesAPIResponse object to check.
|
| 414 |
+
output_type: The type of output to check for.
|
| 415 |
+
|
| 416 |
+
Returns:
|
| 417 |
+
True if the ResponsesAPIResponse includes one of the specified output types, False otherwise.
|
| 418 |
+
"""
|
| 419 |
+
output = response_object.output
|
| 420 |
+
for output_item in output:
|
| 421 |
+
_output_type: Optional[str] = getattr(output_item, "type", None)
|
| 422 |
+
if _output_type == output_type:
|
| 423 |
+
return True
|
| 424 |
+
return False
|
| 425 |
+
|
| 426 |
+
@staticmethod
|
| 427 |
+
def _safe_get_model_info(
|
| 428 |
+
model: str, custom_llm_provider: Optional[str] = None
|
| 429 |
+
) -> Optional[ModelInfo]:
|
| 430 |
+
try:
|
| 431 |
+
return litellm.get_model_info(
|
| 432 |
+
model=model, custom_llm_provider=custom_llm_provider
|
| 433 |
+
)
|
| 434 |
+
except Exception:
|
| 435 |
+
return None
|
| 436 |
+
|
| 437 |
+
@staticmethod
|
| 438 |
+
def get_cost_for_web_search(
|
| 439 |
+
web_search_options: Optional[WebSearchOptions] = None,
|
| 440 |
+
model_info: Optional[ModelInfo] = None,
|
| 441 |
+
) -> float:
|
| 442 |
+
"""
|
| 443 |
+
If request includes `web_search_options`, calculate the cost of the web search.
|
| 444 |
+
"""
|
| 445 |
+
web_search_options = web_search_options or {}
|
| 446 |
+
if model_info is None:
|
| 447 |
+
return 0.0
|
| 448 |
+
|
| 449 |
+
search_context_raw: Any = model_info.get("search_context_cost_per_query", {})
|
| 450 |
+
search_context_pricing: SearchContextCostPerQuery = (
|
| 451 |
+
SearchContextCostPerQuery(**search_context_raw)
|
| 452 |
+
if search_context_raw
|
| 453 |
+
else SearchContextCostPerQuery()
|
| 454 |
+
)
|
| 455 |
+
if web_search_options.get("search_context_size", None) == "low":
|
| 456 |
+
return search_context_pricing.get("search_context_size_low", 0.0)
|
| 457 |
+
elif web_search_options.get("search_context_size", None) == "medium":
|
| 458 |
+
return search_context_pricing.get("search_context_size_medium", 0.0)
|
| 459 |
+
elif web_search_options.get("search_context_size", None) == "high":
|
| 460 |
+
return search_context_pricing.get("search_context_size_high", 0.0)
|
| 461 |
+
return StandardBuiltInToolCostTracking.get_default_cost_for_web_search(
|
| 462 |
+
model_info
|
| 463 |
+
)
|
| 464 |
+
|
| 465 |
+
@staticmethod
|
| 466 |
+
def get_default_cost_for_web_search(
|
| 467 |
+
model_info: Optional[ModelInfo] = None,
|
| 468 |
+
) -> float:
|
| 469 |
+
"""
|
| 470 |
+
If no web search options are provided, use the `search_context_size_medium` pricing.
|
| 471 |
+
|
| 472 |
+
https://platform.openai.com/docs/pricing#web-search
|
| 473 |
+
"""
|
| 474 |
+
if model_info is None:
|
| 475 |
+
return 0.0
|
| 476 |
+
search_context_raw: Any = (
|
| 477 |
+
model_info.get("search_context_cost_per_query", {}) or {}
|
| 478 |
+
)
|
| 479 |
+
search_context_pricing: SearchContextCostPerQuery = (
|
| 480 |
+
SearchContextCostPerQuery(**search_context_raw)
|
| 481 |
+
if search_context_raw
|
| 482 |
+
else SearchContextCostPerQuery()
|
| 483 |
+
)
|
| 484 |
+
return search_context_pricing.get("search_context_size_medium", 0.0)
|
| 485 |
+
|
| 486 |
+
@staticmethod
|
| 487 |
+
def get_cost_for_file_search(
|
| 488 |
+
file_search: Optional[FileSearchTool] = None,
|
| 489 |
+
provider: Optional[str] = None,
|
| 490 |
+
model_info: Optional[dict] = None,
|
| 491 |
+
storage_gb: Optional[float] = None,
|
| 492 |
+
days: Optional[float] = None,
|
| 493 |
+
) -> float:
|
| 494 |
+
""" "
|
| 495 |
+
OpenAI: $2.50/1k calls
|
| 496 |
+
Azure: $0.1 USD per 1 GB/Day (storage-based pricing)
|
| 497 |
+
|
| 498 |
+
Doc: https://platform.openai.com/docs/pricing#built-in-tools
|
| 499 |
+
"""
|
| 500 |
+
if file_search is None:
|
| 501 |
+
return 0.0
|
| 502 |
+
|
| 503 |
+
# Check if model-specific pricing is available
|
| 504 |
+
if (
|
| 505 |
+
model_info
|
| 506 |
+
and "file_search_cost_per_gb_per_day" in model_info
|
| 507 |
+
and provider == "azure"
|
| 508 |
+
):
|
| 509 |
+
if storage_gb and days:
|
| 510 |
+
return storage_gb * days * model_info["file_search_cost_per_gb_per_day"]
|
| 511 |
+
elif model_info and "file_search_cost_per_1k_calls" in model_info:
|
| 512 |
+
return model_info["file_search_cost_per_1k_calls"]
|
| 513 |
+
|
| 514 |
+
# Azure has storage-based pricing for file search
|
| 515 |
+
if provider == "azure":
|
| 516 |
+
from litellm.constants import AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY
|
| 517 |
+
|
| 518 |
+
if storage_gb and days:
|
| 519 |
+
return storage_gb * days * AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY
|
| 520 |
+
# Default to 0 if no storage info provided
|
| 521 |
+
return 0.0
|
| 522 |
+
|
| 523 |
+
# Default to OpenAI pricing (per-call based)
|
| 524 |
+
return OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
|
| 525 |
+
|
| 526 |
+
@staticmethod
|
| 527 |
+
def get_cost_for_vector_store(
|
| 528 |
+
vector_store_usage: Optional[dict] = None,
|
| 529 |
+
provider: Optional[str] = None,
|
| 530 |
+
model_info: Optional[dict] = None,
|
| 531 |
+
) -> float:
|
| 532 |
+
"""
|
| 533 |
+
Calculate cost for vector store usage.
|
| 534 |
+
|
| 535 |
+
Azure charges based on storage size and duration.
|
| 536 |
+
"""
|
| 537 |
+
if vector_store_usage is None:
|
| 538 |
+
return 0.0
|
| 539 |
+
|
| 540 |
+
storage_gb = vector_store_usage.get("storage_gb", 0.0)
|
| 541 |
+
days = vector_store_usage.get("days", 0.0)
|
| 542 |
+
|
| 543 |
+
# Check if model-specific pricing is available
|
| 544 |
+
if model_info and "vector_store_cost_per_gb_per_day" in model_info:
|
| 545 |
+
return storage_gb * days * model_info["vector_store_cost_per_gb_per_day"]
|
| 546 |
+
|
| 547 |
+
# Azure has different pricing structure for vector store
|
| 548 |
+
if provider == "azure":
|
| 549 |
+
from litellm.constants import AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY
|
| 550 |
+
|
| 551 |
+
return storage_gb * days * AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY
|
| 552 |
+
|
| 553 |
+
# OpenAI doesn't charge separately for vector store (included in embeddings)
|
| 554 |
+
return 0.0
|
| 555 |
+
|
| 556 |
+
@staticmethod
|
| 557 |
+
def get_cost_for_computer_use(
|
| 558 |
+
input_tokens: Optional[int] = None,
|
| 559 |
+
output_tokens: Optional[int] = None,
|
| 560 |
+
provider: Optional[str] = None,
|
| 561 |
+
model_info: Optional[dict] = None,
|
| 562 |
+
) -> float:
|
| 563 |
+
"""
|
| 564 |
+
Calculate cost for computer use feature.
|
| 565 |
+
|
| 566 |
+
Azure: $0.003 USD per 1K input tokens, $0.012 USD per 1K output tokens
|
| 567 |
+
"""
|
| 568 |
+
if provider == "azure" and (input_tokens or output_tokens):
|
| 569 |
+
# Check if model-specific pricing is available
|
| 570 |
+
if model_info:
|
| 571 |
+
input_cost = model_info.get(
|
| 572 |
+
"computer_use_input_cost_per_1k_tokens", 0.0
|
| 573 |
+
)
|
| 574 |
+
output_cost = model_info.get(
|
| 575 |
+
"computer_use_output_cost_per_1k_tokens", 0.0
|
| 576 |
+
)
|
| 577 |
+
if input_cost or output_cost:
|
| 578 |
+
total_cost = 0.0
|
| 579 |
+
if input_tokens:
|
| 580 |
+
total_cost += (input_tokens / 1000.0) * input_cost
|
| 581 |
+
if output_tokens:
|
| 582 |
+
total_cost += (output_tokens / 1000.0) * output_cost
|
| 583 |
+
return total_cost
|
| 584 |
+
|
| 585 |
+
# Azure default pricing
|
| 586 |
+
from litellm.constants import (
|
| 587 |
+
AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS,
|
| 588 |
+
AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS,
|
| 589 |
+
)
|
| 590 |
+
|
| 591 |
+
total_cost = 0.0
|
| 592 |
+
if input_tokens:
|
| 593 |
+
total_cost += (
|
| 594 |
+
input_tokens / 1000.0
|
| 595 |
+
) * AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS
|
| 596 |
+
if output_tokens:
|
| 597 |
+
total_cost += (
|
| 598 |
+
output_tokens / 1000.0
|
| 599 |
+
) * AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS
|
| 600 |
+
return total_cost
|
| 601 |
+
|
| 602 |
+
# OpenAI doesn't charge separately for computer use yet
|
| 603 |
+
return 0.0
|
| 604 |
+
|
| 605 |
+
@staticmethod
|
| 606 |
+
def _get_code_interpreter_cost_from_model_map(
|
| 607 |
+
provider: str,
|
| 608 |
+
) -> Optional[float]:
|
| 609 |
+
"""
|
| 610 |
+
Get code interpreter cost per session from model cost map.
|
| 611 |
+
"""
|
| 612 |
+
import litellm
|
| 613 |
+
|
| 614 |
+
try:
|
| 615 |
+
container_model = f"{provider}/container"
|
| 616 |
+
model_info = litellm.get_model_info(
|
| 617 |
+
model=container_model, custom_llm_provider=provider
|
| 618 |
+
)
|
| 619 |
+
model_key = (
|
| 620 |
+
model_info.get("key")
|
| 621 |
+
if isinstance(model_info, dict)
|
| 622 |
+
else getattr(model_info, "key", None)
|
| 623 |
+
)
|
| 624 |
+
|
| 625 |
+
if model_key and model_key in litellm.model_cost:
|
| 626 |
+
return litellm.model_cost[model_key].get(
|
| 627 |
+
"code_interpreter_cost_per_session"
|
| 628 |
+
)
|
| 629 |
+
|
| 630 |
+
except Exception:
|
| 631 |
+
pass
|
| 632 |
+
|
| 633 |
+
return None
|
| 634 |
+
|
| 635 |
+
@staticmethod
|
| 636 |
+
def get_cost_for_code_interpreter(
|
| 637 |
+
sessions: Optional[int] = None,
|
| 638 |
+
provider: Optional[str] = None,
|
| 639 |
+
model_info: Optional[dict] = None,
|
| 640 |
+
) -> float:
|
| 641 |
+
"""
|
| 642 |
+
Calculate cost for code interpreter feature.
|
| 643 |
+
|
| 644 |
+
Azure: $0.03 USD per session (from model cost map)
|
| 645 |
+
OpenAI: $0.03 USD per session (from model cost map)
|
| 646 |
+
"""
|
| 647 |
+
if sessions is None or sessions == 0:
|
| 648 |
+
return 0.0
|
| 649 |
+
|
| 650 |
+
# Check if model-specific pricing is available
|
| 651 |
+
if model_info and "code_interpreter_cost_per_session" in model_info:
|
| 652 |
+
return sessions * model_info["code_interpreter_cost_per_session"]
|
| 653 |
+
|
| 654 |
+
# Try to get cost from model cost map for any provider
|
| 655 |
+
if provider:
|
| 656 |
+
cost_per_session = StandardBuiltInToolCostTracking._get_code_interpreter_cost_from_model_map(
|
| 657 |
+
provider=provider
|
| 658 |
+
)
|
| 659 |
+
if cost_per_session is not None:
|
| 660 |
+
return sessions * cost_per_session
|
| 661 |
+
|
| 662 |
+
return 0.0
|
| 663 |
+
|
| 664 |
+
@staticmethod
|
| 665 |
+
def chat_completion_response_includes_annotations(
|
| 666 |
+
response_object: ModelResponse,
|
| 667 |
+
) -> bool:
|
| 668 |
+
for _choice in response_object.choices:
|
| 669 |
+
message = getattr(_choice, "message", None)
|
| 670 |
+
if (
|
| 671 |
+
message is not None
|
| 672 |
+
and hasattr(message, "annotations")
|
| 673 |
+
and message.annotations is not None
|
| 674 |
+
and len(message.annotations) > 0
|
| 675 |
+
):
|
| 676 |
+
return True
|
| 677 |
+
return False
|
| 678 |
+
|
| 679 |
+
@staticmethod
|
| 680 |
+
def _get_web_search_options(kwargs: Dict) -> Optional[WebSearchOptions]:
|
| 681 |
+
if "web_search_options" in kwargs:
|
| 682 |
+
return WebSearchOptions(**kwargs.get("web_search_options", {}))
|
| 683 |
+
|
| 684 |
+
tools = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
|
| 685 |
+
kwargs=kwargs, tool_type="web_search_preview"
|
| 686 |
+
) or StandardBuiltInToolCostTracking._get_tools_from_kwargs(
|
| 687 |
+
kwargs=kwargs, tool_type="web_search"
|
| 688 |
+
)
|
| 689 |
+
if tools:
|
| 690 |
+
# Look for web search tool in the tools array
|
| 691 |
+
for tool in tools:
|
| 692 |
+
if isinstance(tool, dict):
|
| 693 |
+
if StandardBuiltInToolCostTracking._is_web_search_tool_call(tool):
|
| 694 |
+
return WebSearchOptions(**tool)
|
| 695 |
+
return None
|
| 696 |
+
|
| 697 |
+
@staticmethod
|
| 698 |
+
def _get_tools_from_kwargs(kwargs: Dict, tool_type: str) -> Optional[List[Dict]]:
|
| 699 |
+
if "tools" in kwargs:
|
| 700 |
+
return kwargs.get("tools", [])
|
| 701 |
+
return None
|
| 702 |
+
|
| 703 |
+
@staticmethod
|
| 704 |
+
def _get_file_search_tool_call(kwargs: Dict) -> Optional[FileSearchTool]:
|
| 705 |
+
tools = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
|
| 706 |
+
kwargs, "file_search"
|
| 707 |
+
)
|
| 708 |
+
if tools:
|
| 709 |
+
for tool in tools:
|
| 710 |
+
if isinstance(tool, dict):
|
| 711 |
+
if StandardBuiltInToolCostTracking._is_file_search_tool_call(tool):
|
| 712 |
+
return FileSearchTool(**tool)
|
| 713 |
+
return None
|
| 714 |
+
|
| 715 |
+
@staticmethod
|
| 716 |
+
def _is_web_search_tool_call(tool: Dict) -> bool:
|
| 717 |
+
if tool.get("type", None) == "web_search_preview":
|
| 718 |
+
return True
|
| 719 |
+
if tool.get("type", None) == "web_search":
|
| 720 |
+
return True
|
| 721 |
+
if "search_context_size" in tool:
|
| 722 |
+
return True
|
| 723 |
+
return False
|
| 724 |
+
|
| 725 |
+
@staticmethod
|
| 726 |
+
def _is_file_search_tool_call(tool: Dict) -> bool:
|
| 727 |
+
if tool.get("type", None) == "file_search":
|
| 728 |
+
return True
|
| 729 |
+
return False
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Any, Optional, Union
|
| 2 |
+
|
| 3 |
+
from litellm.types.utils import (
|
| 4 |
+
PromptTokensDetailsWrapper,
|
| 5 |
+
TranscriptionUsageDurationObject,
|
| 6 |
+
TranscriptionUsageTokensObject,
|
| 7 |
+
Usage,
|
| 8 |
+
)
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
class TranscriptionUsageObjectTransformation:
|
| 12 |
+
@staticmethod
|
| 13 |
+
def is_transcription_usage_object(
|
| 14 |
+
usage_object: Any,
|
| 15 |
+
) -> bool:
|
| 16 |
+
return isinstance(usage_object, TranscriptionUsageDurationObject) or isinstance(
|
| 17 |
+
usage_object, TranscriptionUsageTokensObject
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
@staticmethod
|
| 21 |
+
def transform_transcription_usage_object(
|
| 22 |
+
usage_object: Union[
|
| 23 |
+
TranscriptionUsageDurationObject, TranscriptionUsageTokensObject
|
| 24 |
+
],
|
| 25 |
+
) -> Optional[Usage]:
|
| 26 |
+
if isinstance(usage_object, TranscriptionUsageDurationObject):
|
| 27 |
+
return None
|
| 28 |
+
elif isinstance(usage_object, TranscriptionUsageTokensObject):
|
| 29 |
+
return Usage(
|
| 30 |
+
prompt_tokens=usage_object.input_tokens,
|
| 31 |
+
completion_tokens=usage_object.output_tokens,
|
| 32 |
+
total_tokens=usage_object.total_tokens,
|
| 33 |
+
prompt_tokens_details=PromptTokensDetailsWrapper(
|
| 34 |
+
text_tokens=usage_object.input_token_details.text_tokens,
|
| 35 |
+
audio_tokens=usage_object.input_token_details.audio_tokens,
|
| 36 |
+
),
|
| 37 |
+
)
|
| 38 |
+
return None
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/utils.py
ADDED
|
@@ -0,0 +1,1034 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# What is this?
|
| 2 |
+
## Helper utilities for cost_per_token()
|
| 3 |
+
|
| 4 |
+
from typing import Literal, Optional, Tuple, TypedDict, cast
|
| 5 |
+
|
| 6 |
+
import litellm
|
| 7 |
+
from litellm._logging import verbose_logger
|
| 8 |
+
from litellm.types.utils import (
|
| 9 |
+
CacheCreationTokenDetails,
|
| 10 |
+
CallTypes,
|
| 11 |
+
CompletionTokensDetailsWrapper,
|
| 12 |
+
ImageResponse,
|
| 13 |
+
ModelInfo,
|
| 14 |
+
PassthroughCallTypes,
|
| 15 |
+
PromptTokensDetailsWrapper,
|
| 16 |
+
ServiceTier,
|
| 17 |
+
Usage,
|
| 18 |
+
)
|
| 19 |
+
from litellm.utils import get_model_info
|
| 20 |
+
|
| 21 |
+
# Pre-resolved CallTypes enum values for fast membership checks
|
| 22 |
+
_IMAGE_RESPONSE_CALL_TYPES = frozenset(
|
| 23 |
+
{
|
| 24 |
+
CallTypes.image_generation.value,
|
| 25 |
+
CallTypes.aimage_generation.value,
|
| 26 |
+
PassthroughCallTypes.passthrough_image_generation.value,
|
| 27 |
+
CallTypes.image_edit.value,
|
| 28 |
+
CallTypes.aimage_edit.value,
|
| 29 |
+
}
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _is_above_128k(tokens: float) -> bool:
|
| 34 |
+
if tokens > 128000:
|
| 35 |
+
return True
|
| 36 |
+
return False
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def get_billable_input_tokens(usage: Usage) -> int:
|
| 40 |
+
"""
|
| 41 |
+
Returns the number of billable input tokens.
|
| 42 |
+
Subtracts cached tokens from prompt tokens if applicable.
|
| 43 |
+
"""
|
| 44 |
+
details = _parse_prompt_tokens_details(usage)
|
| 45 |
+
return usage.prompt_tokens - details["cache_hit_tokens"]
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def select_cost_metric_for_model(
|
| 49 |
+
model_info: ModelInfo,
|
| 50 |
+
) -> Literal["cost_per_character", "cost_per_token"]:
|
| 51 |
+
"""
|
| 52 |
+
Select 'cost_per_character' if model_info has 'input_cost_per_character'
|
| 53 |
+
Select 'cost_per_token' if model_info has 'input_cost_per_token'
|
| 54 |
+
"""
|
| 55 |
+
if model_info.get("input_cost_per_character"):
|
| 56 |
+
return "cost_per_character"
|
| 57 |
+
elif model_info.get("input_cost_per_token"):
|
| 58 |
+
return "cost_per_token"
|
| 59 |
+
else:
|
| 60 |
+
raise ValueError(
|
| 61 |
+
f"Model {model_info['key']} does not have 'input_cost_per_character' or 'input_cost_per_token'"
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _generic_cost_per_character(
|
| 66 |
+
model: str,
|
| 67 |
+
custom_llm_provider: str,
|
| 68 |
+
prompt_characters: float,
|
| 69 |
+
completion_characters: float,
|
| 70 |
+
custom_prompt_cost: Optional[float],
|
| 71 |
+
custom_completion_cost: Optional[float],
|
| 72 |
+
) -> Tuple[Optional[float], Optional[float]]:
|
| 73 |
+
"""
|
| 74 |
+
Calculates cost per character for aspeech/speech calls.
|
| 75 |
+
|
| 76 |
+
Calculates the cost per character for a given model, input messages, and response object.
|
| 77 |
+
|
| 78 |
+
Input:
|
| 79 |
+
- model: str, the model name without provider prefix
|
| 80 |
+
- custom_llm_provider: str, "vertex_ai-*"
|
| 81 |
+
- prompt_characters: float, the number of input characters
|
| 82 |
+
- completion_characters: float, the number of output characters
|
| 83 |
+
|
| 84 |
+
Returns:
|
| 85 |
+
Tuple[Optional[float], Optional[float]] - prompt_cost_in_usd, completion_cost_in_usd.
|
| 86 |
+
- returns None if not able to calculate cost.
|
| 87 |
+
|
| 88 |
+
Raises:
|
| 89 |
+
Exception if 'input_cost_per_character' or 'output_cost_per_character' is missing from model_info
|
| 90 |
+
"""
|
| 91 |
+
## GET MODEL INFO
|
| 92 |
+
model_info = litellm.get_model_info(
|
| 93 |
+
model=model, custom_llm_provider=custom_llm_provider
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
## CALCULATE INPUT COST
|
| 97 |
+
try:
|
| 98 |
+
if custom_prompt_cost is None:
|
| 99 |
+
assert (
|
| 100 |
+
"input_cost_per_character" in model_info
|
| 101 |
+
and model_info["input_cost_per_character"] is not None
|
| 102 |
+
), "model info for model={} does not have 'input_cost_per_character'-pricing\nmodel_info={}".format(
|
| 103 |
+
model, model_info
|
| 104 |
+
)
|
| 105 |
+
custom_prompt_cost = model_info["input_cost_per_character"]
|
| 106 |
+
|
| 107 |
+
prompt_cost = prompt_characters * custom_prompt_cost
|
| 108 |
+
except Exception as e:
|
| 109 |
+
verbose_logger.exception(
|
| 110 |
+
"litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\nDefaulting to None".format(
|
| 111 |
+
str(e)
|
| 112 |
+
)
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
prompt_cost = None
|
| 116 |
+
|
| 117 |
+
## CALCULATE OUTPUT COST
|
| 118 |
+
try:
|
| 119 |
+
if custom_completion_cost is None:
|
| 120 |
+
assert (
|
| 121 |
+
"output_cost_per_character" in model_info
|
| 122 |
+
and model_info["output_cost_per_character"] is not None
|
| 123 |
+
), "model info for model={} does not have 'output_cost_per_character'-pricing\nmodel_info={}".format(
|
| 124 |
+
model, model_info
|
| 125 |
+
)
|
| 126 |
+
custom_completion_cost = model_info["output_cost_per_character"]
|
| 127 |
+
completion_cost = completion_characters * custom_completion_cost
|
| 128 |
+
except Exception as e:
|
| 129 |
+
verbose_logger.exception(
|
| 130 |
+
"litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\nDefaulting to None".format(
|
| 131 |
+
str(e)
|
| 132 |
+
)
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
completion_cost = None
|
| 136 |
+
|
| 137 |
+
return prompt_cost, completion_cost
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> str:
|
| 141 |
+
"""
|
| 142 |
+
Get the appropriate cost key based on service tier.
|
| 143 |
+
|
| 144 |
+
Args:
|
| 145 |
+
base_key: The base cost key (e.g., "input_cost_per_token")
|
| 146 |
+
service_tier: The service tier ("flex", "priority", or None for standard)
|
| 147 |
+
|
| 148 |
+
Returns:
|
| 149 |
+
str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token")
|
| 150 |
+
"""
|
| 151 |
+
if service_tier is None:
|
| 152 |
+
return base_key
|
| 153 |
+
|
| 154 |
+
# Only use service tier specific keys for "flex" and "priority"
|
| 155 |
+
if service_tier.lower() in [ServiceTier.FLEX.value, ServiceTier.PRIORITY.value]:
|
| 156 |
+
return f"{base_key}_{service_tier.lower()}"
|
| 157 |
+
|
| 158 |
+
# For any other service tier, use standard pricing
|
| 159 |
+
return base_key
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def _get_token_base_cost(
|
| 163 |
+
model_info: ModelInfo, usage: Usage, service_tier: Optional[str] = None
|
| 164 |
+
) -> Tuple[float, float, float, float, float]:
|
| 165 |
+
"""
|
| 166 |
+
Return prompt cost, completion cost, and cache costs for a given model and usage.
|
| 167 |
+
|
| 168 |
+
If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set,
|
| 169 |
+
then we use the corresponding threshold cost for all token types.
|
| 170 |
+
|
| 171 |
+
Returns:
|
| 172 |
+
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
|
| 173 |
+
"""
|
| 174 |
+
# Get service tier aware cost keys
|
| 175 |
+
input_cost_key = _get_service_tier_cost_key("input_cost_per_token", service_tier)
|
| 176 |
+
output_cost_key = _get_service_tier_cost_key("output_cost_per_token", service_tier)
|
| 177 |
+
cache_creation_cost_key = _get_service_tier_cost_key(
|
| 178 |
+
"cache_creation_input_token_cost", service_tier
|
| 179 |
+
)
|
| 180 |
+
cache_read_cost_key = _get_service_tier_cost_key(
|
| 181 |
+
"cache_read_input_token_cost", service_tier
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
prompt_base_cost = cast(float, _get_cost_per_unit(model_info, input_cost_key))
|
| 185 |
+
completion_base_cost = cast(float, _get_cost_per_unit(model_info, output_cost_key))
|
| 186 |
+
|
| 187 |
+
# For image generation models that don't have output_cost_per_token,
|
| 188 |
+
# use output_cost_per_image_token as the base cost (all output tokens are image tokens)
|
| 189 |
+
if completion_base_cost == 0.0 or completion_base_cost is None:
|
| 190 |
+
output_image_cost = _get_cost_per_unit(
|
| 191 |
+
model_info, "output_cost_per_image_token", None
|
| 192 |
+
)
|
| 193 |
+
if output_image_cost is not None:
|
| 194 |
+
completion_base_cost = cast(float, output_image_cost)
|
| 195 |
+
cache_creation_cost = cast(
|
| 196 |
+
float, _get_cost_per_unit(model_info, cache_creation_cost_key)
|
| 197 |
+
)
|
| 198 |
+
cache_creation_cost_above_1hr = cast(
|
| 199 |
+
float,
|
| 200 |
+
_get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),
|
| 201 |
+
)
|
| 202 |
+
cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key))
|
| 203 |
+
|
| 204 |
+
## CHECK IF ABOVE THRESHOLD
|
| 205 |
+
# Optimization: collect threshold keys first to avoid sorting all model_info keys.
|
| 206 |
+
# Most models don't have threshold pricing, so we can return early.
|
| 207 |
+
# Exclude service_tier-specific variants (e.g. input_cost_per_token_above_200k_tokens_priority)
|
| 208 |
+
# so that the threshold detection loop only processes standard keys. The
|
| 209 |
+
# service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key.
|
| 210 |
+
threshold_keys = [
|
| 211 |
+
k
|
| 212 |
+
for k in model_info
|
| 213 |
+
if k.startswith("input_cost_per_token_above_")
|
| 214 |
+
and not any(k.endswith(f"_{st.value}") for st in ServiceTier)
|
| 215 |
+
]
|
| 216 |
+
if not threshold_keys:
|
| 217 |
+
return (
|
| 218 |
+
prompt_base_cost,
|
| 219 |
+
completion_base_cost,
|
| 220 |
+
cache_creation_cost,
|
| 221 |
+
cache_creation_cost_above_1hr,
|
| 222 |
+
cache_read_cost,
|
| 223 |
+
)
|
| 224 |
+
|
| 225 |
+
# Only sort the threshold keys (typically 1-2 keys instead of 66+)
|
| 226 |
+
threshold: Optional[float] = None
|
| 227 |
+
for key in sorted(threshold_keys, reverse=True):
|
| 228 |
+
value = model_info.get(key)
|
| 229 |
+
if value is not None:
|
| 230 |
+
try:
|
| 231 |
+
# Handle both formats: _above_128k_tokens and _above_128_tokens
|
| 232 |
+
threshold_str = key.split("_above_")[1].split("_tokens")[0]
|
| 233 |
+
threshold = float(threshold_str.replace("k", "")) * (
|
| 234 |
+
1000 if "k" in threshold_str else 1
|
| 235 |
+
)
|
| 236 |
+
if usage.prompt_tokens > threshold:
|
| 237 |
+
# Prefer a service_tier-specific above-threshold key when available,
|
| 238 |
+
# e.g. input_cost_per_token_priority_above_200k_tokens for Gemini
|
| 239 |
+
# ON_DEMAND_PRIORITY. Falls back to the standard key automatically
|
| 240 |
+
# via _get_cost_per_unit's service_tier fallback logic.
|
| 241 |
+
tiered_input_key = (
|
| 242 |
+
_get_service_tier_cost_key(
|
| 243 |
+
f"input_cost_per_token_above_{threshold_str}_tokens",
|
| 244 |
+
service_tier,
|
| 245 |
+
)
|
| 246 |
+
if service_tier
|
| 247 |
+
else key
|
| 248 |
+
)
|
| 249 |
+
prompt_base_cost = cast(
|
| 250 |
+
float,
|
| 251 |
+
_get_cost_per_unit(
|
| 252 |
+
model_info, tiered_input_key, prompt_base_cost
|
| 253 |
+
),
|
| 254 |
+
)
|
| 255 |
+
tiered_output_key = (
|
| 256 |
+
_get_service_tier_cost_key(
|
| 257 |
+
f"output_cost_per_token_above_{threshold_str}_tokens",
|
| 258 |
+
service_tier,
|
| 259 |
+
)
|
| 260 |
+
if service_tier
|
| 261 |
+
else f"output_cost_per_token_above_{threshold_str}_tokens"
|
| 262 |
+
)
|
| 263 |
+
completion_base_cost = cast(
|
| 264 |
+
float,
|
| 265 |
+
_get_cost_per_unit(
|
| 266 |
+
model_info,
|
| 267 |
+
tiered_output_key,
|
| 268 |
+
completion_base_cost,
|
| 269 |
+
),
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
# Apply tiered pricing to cache costs
|
| 273 |
+
cache_creation_tiered_key = (
|
| 274 |
+
f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
|
| 275 |
+
)
|
| 276 |
+
cache_creation_1hr_tiered_key = f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
|
| 277 |
+
cache_read_tiered_key = (
|
| 278 |
+
f"cache_read_input_token_cost_above_{threshold_str}_tokens"
|
| 279 |
+
)
|
| 280 |
+
|
| 281 |
+
if cache_creation_tiered_key in model_info:
|
| 282 |
+
cache_creation_cost = cast(
|
| 283 |
+
float,
|
| 284 |
+
_get_cost_per_unit(
|
| 285 |
+
model_info,
|
| 286 |
+
cache_creation_tiered_key,
|
| 287 |
+
cache_creation_cost,
|
| 288 |
+
),
|
| 289 |
+
)
|
| 290 |
+
|
| 291 |
+
if cache_creation_1hr_tiered_key in model_info:
|
| 292 |
+
cache_creation_cost_above_1hr = cast(
|
| 293 |
+
float,
|
| 294 |
+
_get_cost_per_unit(
|
| 295 |
+
model_info,
|
| 296 |
+
cache_creation_1hr_tiered_key,
|
| 297 |
+
cache_creation_cost_above_1hr,
|
| 298 |
+
),
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
if cache_read_tiered_key in model_info:
|
| 302 |
+
cache_read_cost = cast(
|
| 303 |
+
float,
|
| 304 |
+
_get_cost_per_unit(
|
| 305 |
+
model_info, cache_read_tiered_key, cache_read_cost
|
| 306 |
+
),
|
| 307 |
+
)
|
| 308 |
+
|
| 309 |
+
break
|
| 310 |
+
except (IndexError, ValueError):
|
| 311 |
+
continue
|
| 312 |
+
except Exception:
|
| 313 |
+
continue
|
| 314 |
+
|
| 315 |
+
return (
|
| 316 |
+
prompt_base_cost,
|
| 317 |
+
completion_base_cost,
|
| 318 |
+
cache_creation_cost,
|
| 319 |
+
cache_creation_cost_above_1hr,
|
| 320 |
+
cache_read_cost,
|
| 321 |
+
)
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
def calculate_cost_component(
|
| 325 |
+
model_info: ModelInfo, cost_key: str, usage_value: Optional[float]
|
| 326 |
+
) -> float:
|
| 327 |
+
"""
|
| 328 |
+
Generic cost calculator for any usage component
|
| 329 |
+
|
| 330 |
+
Args:
|
| 331 |
+
model_info: Dictionary containing cost information
|
| 332 |
+
cost_key: The key for the cost multiplier in model_info (e.g., 'input_cost_per_audio_token')
|
| 333 |
+
usage_value: The actual usage value (e.g., number of tokens, characters, seconds)
|
| 334 |
+
|
| 335 |
+
Returns:
|
| 336 |
+
float: The calculated cost
|
| 337 |
+
"""
|
| 338 |
+
cost_per_unit = _get_cost_per_unit(model_info, cost_key)
|
| 339 |
+
if (
|
| 340 |
+
cost_per_unit is not None
|
| 341 |
+
and isinstance(cost_per_unit, float)
|
| 342 |
+
and usage_value is not None
|
| 343 |
+
and usage_value > 0
|
| 344 |
+
):
|
| 345 |
+
return float(usage_value) * cost_per_unit
|
| 346 |
+
return 0.0
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def _get_cost_per_unit(
|
| 350 |
+
model_info: ModelInfo, cost_key: str, default_value: Optional[float] = 0.0
|
| 351 |
+
) -> Optional[float]:
|
| 352 |
+
# Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
|
| 353 |
+
cost_per_unit = model_info.get(cost_key)
|
| 354 |
+
if isinstance(cost_per_unit, float):
|
| 355 |
+
return cost_per_unit
|
| 356 |
+
if isinstance(cost_per_unit, int):
|
| 357 |
+
return float(cost_per_unit)
|
| 358 |
+
if isinstance(cost_per_unit, str):
|
| 359 |
+
try:
|
| 360 |
+
return float(cost_per_unit)
|
| 361 |
+
except ValueError:
|
| 362 |
+
verbose_logger.exception(
|
| 363 |
+
f"litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - {cost_per_unit}\nDefaulting to 0.0"
|
| 364 |
+
)
|
| 365 |
+
|
| 366 |
+
# If the service tier key doesn't exist or is None, try to fall back to the standard key
|
| 367 |
+
if cost_per_unit is None:
|
| 368 |
+
# Check if any service tier suffix exists in the cost key using ServiceTier enum
|
| 369 |
+
for service_tier in ServiceTier:
|
| 370 |
+
suffix = f"_{service_tier.value}"
|
| 371 |
+
if suffix in cost_key:
|
| 372 |
+
# Extract the base key by removing the matched suffix
|
| 373 |
+
base_key = cost_key.replace(suffix, "")
|
| 374 |
+
fallback_cost = model_info.get(base_key)
|
| 375 |
+
if isinstance(fallback_cost, float):
|
| 376 |
+
return fallback_cost
|
| 377 |
+
if isinstance(fallback_cost, int):
|
| 378 |
+
return float(fallback_cost)
|
| 379 |
+
if isinstance(fallback_cost, str):
|
| 380 |
+
try:
|
| 381 |
+
return float(fallback_cost)
|
| 382 |
+
except ValueError:
|
| 383 |
+
verbose_logger.exception(
|
| 384 |
+
f"litellm.litellm_core_utils.llm_cost_calc.utils.py::_get_cost_per_unit(): Exception occured - {fallback_cost}\nDefaulting to 0.0"
|
| 385 |
+
)
|
| 386 |
+
break # Only try the first matching suffix
|
| 387 |
+
|
| 388 |
+
return default_value
|
| 389 |
+
|
| 390 |
+
|
| 391 |
+
def calculate_cache_writing_cost(
|
| 392 |
+
cache_creation_tokens: int,
|
| 393 |
+
cache_creation_token_details: Optional[CacheCreationTokenDetails],
|
| 394 |
+
cache_creation_cost_above_1hr: float,
|
| 395 |
+
cache_creation_cost: float,
|
| 396 |
+
) -> float:
|
| 397 |
+
"""
|
| 398 |
+
Adjust cost of cache creation tokens based on the cache creation token details.
|
| 399 |
+
"""
|
| 400 |
+
total_cost: float = 0.0
|
| 401 |
+
if cache_creation_token_details is not None:
|
| 402 |
+
# get the number of 5m and 1h cache creation tokens
|
| 403 |
+
cache_creation_tokens_5m = (
|
| 404 |
+
cache_creation_token_details.ephemeral_5m_input_tokens
|
| 405 |
+
)
|
| 406 |
+
cache_creation_tokens_1h = (
|
| 407 |
+
cache_creation_token_details.ephemeral_1h_input_tokens
|
| 408 |
+
)
|
| 409 |
+
# add the number of 5m and 1h cache creation tokens to the cache creation tokens
|
| 410 |
+
total_cost += (
|
| 411 |
+
cache_creation_tokens_5m * cache_creation_cost
|
| 412 |
+
if cache_creation_tokens_5m is not None
|
| 413 |
+
else 0.0
|
| 414 |
+
)
|
| 415 |
+
total_cost += (
|
| 416 |
+
cache_creation_tokens_1h * cache_creation_cost_above_1hr
|
| 417 |
+
if cache_creation_tokens_1h is not None
|
| 418 |
+
else 0.0
|
| 419 |
+
)
|
| 420 |
+
else:
|
| 421 |
+
total_cost += cache_creation_tokens * cache_creation_cost
|
| 422 |
+
return total_cost
|
| 423 |
+
|
| 424 |
+
|
| 425 |
+
class PromptTokensDetailsResult(TypedDict):
|
| 426 |
+
cache_hit_tokens: int
|
| 427 |
+
cache_creation_tokens: int
|
| 428 |
+
cache_creation_token_details: Optional[CacheCreationTokenDetails]
|
| 429 |
+
text_tokens: int
|
| 430 |
+
audio_tokens: int
|
| 431 |
+
image_tokens: int
|
| 432 |
+
character_count: int
|
| 433 |
+
image_count: int
|
| 434 |
+
video_length_seconds: float
|
| 435 |
+
|
| 436 |
+
|
| 437 |
+
def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
|
| 438 |
+
cache_hit_tokens = (
|
| 439 |
+
cast(Optional[int], getattr(usage.prompt_tokens_details, "cached_tokens", 0))
|
| 440 |
+
or 0
|
| 441 |
+
)
|
| 442 |
+
cache_creation_tokens = (
|
| 443 |
+
cast(
|
| 444 |
+
Optional[int],
|
| 445 |
+
getattr(usage.prompt_tokens_details, "cache_creation_tokens", 0),
|
| 446 |
+
)
|
| 447 |
+
or 0
|
| 448 |
+
)
|
| 449 |
+
cache_creation_token_details = (
|
| 450 |
+
cast(
|
| 451 |
+
Optional[CacheCreationTokenDetails],
|
| 452 |
+
getattr(usage.prompt_tokens_details, "cache_creation_token_details", None),
|
| 453 |
+
)
|
| 454 |
+
or None
|
| 455 |
+
)
|
| 456 |
+
text_tokens = (
|
| 457 |
+
cast(Optional[int], getattr(usage.prompt_tokens_details, "text_tokens", None))
|
| 458 |
+
or 0 # default to prompt tokens, if this field is not set
|
| 459 |
+
)
|
| 460 |
+
audio_tokens = (
|
| 461 |
+
cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
|
| 462 |
+
or 0
|
| 463 |
+
)
|
| 464 |
+
image_tokens = (
|
| 465 |
+
cast(Optional[int], getattr(usage.prompt_tokens_details, "image_tokens", 0))
|
| 466 |
+
or 0
|
| 467 |
+
)
|
| 468 |
+
character_count = (
|
| 469 |
+
cast(
|
| 470 |
+
Optional[int],
|
| 471 |
+
getattr(usage.prompt_tokens_details, "character_count", 0),
|
| 472 |
+
)
|
| 473 |
+
or 0
|
| 474 |
+
)
|
| 475 |
+
image_count = (
|
| 476 |
+
cast(Optional[int], getattr(usage.prompt_tokens_details, "image_count", 0)) or 0
|
| 477 |
+
)
|
| 478 |
+
video_length_seconds = (
|
| 479 |
+
cast(
|
| 480 |
+
Optional[float],
|
| 481 |
+
getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
|
| 482 |
+
)
|
| 483 |
+
or 0.0
|
| 484 |
+
)
|
| 485 |
+
|
| 486 |
+
return PromptTokensDetailsResult(
|
| 487 |
+
cache_hit_tokens=cache_hit_tokens,
|
| 488 |
+
cache_creation_tokens=cache_creation_tokens,
|
| 489 |
+
cache_creation_token_details=cache_creation_token_details,
|
| 490 |
+
text_tokens=text_tokens,
|
| 491 |
+
audio_tokens=audio_tokens,
|
| 492 |
+
image_tokens=image_tokens,
|
| 493 |
+
character_count=character_count,
|
| 494 |
+
image_count=image_count,
|
| 495 |
+
video_length_seconds=float(video_length_seconds),
|
| 496 |
+
)
|
| 497 |
+
|
| 498 |
+
|
| 499 |
+
class CompletionTokensDetailsResult(TypedDict):
|
| 500 |
+
audio_tokens: int
|
| 501 |
+
text_tokens: int
|
| 502 |
+
reasoning_tokens: int
|
| 503 |
+
image_tokens: int
|
| 504 |
+
|
| 505 |
+
|
| 506 |
+
def _parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResult:
|
| 507 |
+
audio_tokens = (
|
| 508 |
+
cast(
|
| 509 |
+
Optional[int],
|
| 510 |
+
getattr(usage.completion_tokens_details, "audio_tokens", 0),
|
| 511 |
+
)
|
| 512 |
+
or 0
|
| 513 |
+
)
|
| 514 |
+
text_tokens = (
|
| 515 |
+
cast(
|
| 516 |
+
Optional[int],
|
| 517 |
+
getattr(usage.completion_tokens_details, "text_tokens", None),
|
| 518 |
+
)
|
| 519 |
+
or 0 # default to completion tokens, if this field is not set
|
| 520 |
+
)
|
| 521 |
+
reasoning_tokens = (
|
| 522 |
+
cast(
|
| 523 |
+
Optional[int],
|
| 524 |
+
getattr(usage.completion_tokens_details, "reasoning_tokens", 0),
|
| 525 |
+
)
|
| 526 |
+
or 0
|
| 527 |
+
)
|
| 528 |
+
image_tokens = (
|
| 529 |
+
cast(
|
| 530 |
+
Optional[int],
|
| 531 |
+
getattr(usage.completion_tokens_details, "image_tokens", 0),
|
| 532 |
+
)
|
| 533 |
+
or 0
|
| 534 |
+
)
|
| 535 |
+
|
| 536 |
+
return CompletionTokensDetailsResult(
|
| 537 |
+
audio_tokens=audio_tokens,
|
| 538 |
+
text_tokens=text_tokens,
|
| 539 |
+
reasoning_tokens=reasoning_tokens,
|
| 540 |
+
image_tokens=image_tokens,
|
| 541 |
+
)
|
| 542 |
+
|
| 543 |
+
|
| 544 |
+
def _calculate_input_cost(
|
| 545 |
+
prompt_tokens_details: PromptTokensDetailsResult,
|
| 546 |
+
model_info: ModelInfo,
|
| 547 |
+
prompt_base_cost: float,
|
| 548 |
+
cache_read_cost: float,
|
| 549 |
+
cache_creation_cost: float,
|
| 550 |
+
cache_creation_cost_above_1hr: float,
|
| 551 |
+
service_tier: Optional[str] = None,
|
| 552 |
+
) -> float:
|
| 553 |
+
"""
|
| 554 |
+
Calculates the input cost for a given model, prompt tokens, and completion tokens.
|
| 555 |
+
"""
|
| 556 |
+
prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
|
| 557 |
+
|
| 558 |
+
### CACHE READ COST - Now uses tiered pricing
|
| 559 |
+
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
|
| 560 |
+
|
| 561 |
+
### AUDIO COST
|
| 562 |
+
if prompt_tokens_details["audio_tokens"]:
|
| 563 |
+
audio_cost_key = _get_service_tier_cost_key(
|
| 564 |
+
"input_cost_per_audio_token", service_tier
|
| 565 |
+
)
|
| 566 |
+
prompt_cost += calculate_cost_component(
|
| 567 |
+
model_info, audio_cost_key, prompt_tokens_details["audio_tokens"]
|
| 568 |
+
)
|
| 569 |
+
|
| 570 |
+
### IMAGE TOKEN COST
|
| 571 |
+
if prompt_tokens_details["image_tokens"]:
|
| 572 |
+
# For image token costs:
|
| 573 |
+
# First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
|
| 574 |
+
image_token_cost_key = "input_cost_per_image_token"
|
| 575 |
+
if model_info.get(image_token_cost_key) is None:
|
| 576 |
+
image_token_cost_key = "input_cost_per_token"
|
| 577 |
+
prompt_cost += calculate_cost_component(
|
| 578 |
+
model_info, image_token_cost_key, prompt_tokens_details["image_tokens"]
|
| 579 |
+
)
|
| 580 |
+
|
| 581 |
+
### CACHE WRITING COST - Now uses tiered pricing
|
| 582 |
+
if (
|
| 583 |
+
prompt_tokens_details["cache_creation_tokens"]
|
| 584 |
+
or prompt_tokens_details["cache_creation_token_details"] is not None
|
| 585 |
+
):
|
| 586 |
+
prompt_cost += calculate_cache_writing_cost(
|
| 587 |
+
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
|
| 588 |
+
cache_creation_token_details=prompt_tokens_details[
|
| 589 |
+
"cache_creation_token_details"
|
| 590 |
+
],
|
| 591 |
+
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
|
| 592 |
+
cache_creation_cost=cache_creation_cost,
|
| 593 |
+
)
|
| 594 |
+
|
| 595 |
+
### CHARACTER COST
|
| 596 |
+
if prompt_tokens_details["character_count"]:
|
| 597 |
+
prompt_cost += calculate_cost_component(
|
| 598 |
+
model_info,
|
| 599 |
+
"input_cost_per_character",
|
| 600 |
+
prompt_tokens_details["character_count"],
|
| 601 |
+
)
|
| 602 |
+
|
| 603 |
+
### IMAGE COUNT COST
|
| 604 |
+
if prompt_tokens_details["image_count"]:
|
| 605 |
+
prompt_cost += calculate_cost_component(
|
| 606 |
+
model_info, "input_cost_per_image", prompt_tokens_details["image_count"]
|
| 607 |
+
)
|
| 608 |
+
|
| 609 |
+
### VIDEO LENGTH COST
|
| 610 |
+
if prompt_tokens_details["video_length_seconds"]:
|
| 611 |
+
prompt_cost += calculate_cost_component(
|
| 612 |
+
model_info,
|
| 613 |
+
"input_cost_per_video_per_second",
|
| 614 |
+
prompt_tokens_details["video_length_seconds"],
|
| 615 |
+
)
|
| 616 |
+
|
| 617 |
+
return prompt_cost
|
| 618 |
+
|
| 619 |
+
|
| 620 |
+
def generic_cost_per_token( # noqa: PLR0915
|
| 621 |
+
model: str,
|
| 622 |
+
usage: Usage,
|
| 623 |
+
custom_llm_provider: str,
|
| 624 |
+
service_tier: Optional[str] = None,
|
| 625 |
+
) -> Tuple[float, float]:
|
| 626 |
+
"""
|
| 627 |
+
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
|
| 628 |
+
|
| 629 |
+
Handles context caching as well.
|
| 630 |
+
|
| 631 |
+
Input:
|
| 632 |
+
- model: str, the model name without provider prefix
|
| 633 |
+
- usage: LiteLLM Usage block, containing anthropic caching information
|
| 634 |
+
|
| 635 |
+
Returns:
|
| 636 |
+
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
| 637 |
+
"""
|
| 638 |
+
|
| 639 |
+
## GET MODEL INFO
|
| 640 |
+
model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
| 641 |
+
|
| 642 |
+
## CALCULATE INPUT COST
|
| 643 |
+
### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing)
|
| 644 |
+
prompt_cost = 0.0
|
| 645 |
+
### PROCESSING COST
|
| 646 |
+
prompt_tokens_details = PromptTokensDetailsResult(
|
| 647 |
+
cache_hit_tokens=0,
|
| 648 |
+
cache_creation_tokens=0,
|
| 649 |
+
cache_creation_token_details=None,
|
| 650 |
+
text_tokens=usage.prompt_tokens,
|
| 651 |
+
audio_tokens=0,
|
| 652 |
+
image_tokens=0,
|
| 653 |
+
character_count=0,
|
| 654 |
+
image_count=0,
|
| 655 |
+
video_length_seconds=0.0,
|
| 656 |
+
)
|
| 657 |
+
if usage.prompt_tokens_details:
|
| 658 |
+
prompt_tokens_details = _parse_prompt_tokens_details(usage)
|
| 659 |
+
|
| 660 |
+
## EDGE CASE - text tokens not set or includes cached tokens (double-counting)
|
| 661 |
+
## Some providers (like xAI) report text_tokens = prompt_tokens (including cached)
|
| 662 |
+
## We detect this when: text_tokens + cached_tokens + other > prompt_tokens
|
| 663 |
+
## Ref: https://github.com/BerriAI/litellm/issues/19680, #14874, #14875
|
| 664 |
+
|
| 665 |
+
cache_hit = prompt_tokens_details["cache_hit_tokens"]
|
| 666 |
+
text_tokens = prompt_tokens_details["text_tokens"]
|
| 667 |
+
audio_tokens = prompt_tokens_details["audio_tokens"]
|
| 668 |
+
cache_creation = prompt_tokens_details["cache_creation_tokens"]
|
| 669 |
+
image_tokens = prompt_tokens_details["image_tokens"]
|
| 670 |
+
|
| 671 |
+
# Check for double-counting: sum of details > prompt_tokens means overlap
|
| 672 |
+
total_details = (
|
| 673 |
+
text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens
|
| 674 |
+
)
|
| 675 |
+
has_double_counting = cache_hit > 0 and total_details > usage.prompt_tokens
|
| 676 |
+
|
| 677 |
+
if (
|
| 678 |
+
text_tokens == 0 and prompt_tokens_details["image_count"] == 0
|
| 679 |
+
) or has_double_counting:
|
| 680 |
+
text_tokens = (
|
| 681 |
+
usage.prompt_tokens
|
| 682 |
+
- cache_hit
|
| 683 |
+
- audio_tokens
|
| 684 |
+
- cache_creation
|
| 685 |
+
- image_tokens
|
| 686 |
+
)
|
| 687 |
+
prompt_tokens_details["text_tokens"] = text_tokens
|
| 688 |
+
|
| 689 |
+
(
|
| 690 |
+
prompt_base_cost,
|
| 691 |
+
completion_base_cost,
|
| 692 |
+
cache_creation_cost,
|
| 693 |
+
cache_creation_cost_above_1hr,
|
| 694 |
+
cache_read_cost,
|
| 695 |
+
) = _get_token_base_cost(
|
| 696 |
+
model_info=model_info, usage=usage, service_tier=service_tier
|
| 697 |
+
)
|
| 698 |
+
|
| 699 |
+
prompt_cost = _calculate_input_cost(
|
| 700 |
+
prompt_tokens_details=prompt_tokens_details,
|
| 701 |
+
model_info=model_info,
|
| 702 |
+
prompt_base_cost=prompt_base_cost,
|
| 703 |
+
cache_read_cost=cache_read_cost,
|
| 704 |
+
cache_creation_cost=cache_creation_cost,
|
| 705 |
+
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
|
| 706 |
+
service_tier=service_tier,
|
| 707 |
+
)
|
| 708 |
+
|
| 709 |
+
## CALCULATE OUTPUT COST
|
| 710 |
+
text_tokens = 0
|
| 711 |
+
audio_tokens = 0
|
| 712 |
+
reasoning_tokens = 0
|
| 713 |
+
image_tokens = 0
|
| 714 |
+
is_text_tokens_total = False
|
| 715 |
+
if usage.completion_tokens_details is not None:
|
| 716 |
+
completion_tokens_details = _parse_completion_tokens_details(usage)
|
| 717 |
+
audio_tokens = completion_tokens_details["audio_tokens"]
|
| 718 |
+
text_tokens = completion_tokens_details["text_tokens"]
|
| 719 |
+
reasoning_tokens = completion_tokens_details["reasoning_tokens"]
|
| 720 |
+
image_tokens = completion_tokens_details["image_tokens"]
|
| 721 |
+
|
| 722 |
+
# Handle text_tokens calculation:
|
| 723 |
+
# 1. If text_tokens is explicitly provided and > 0, use it
|
| 724 |
+
# 2. If there's a breakdown (reasoning/audio/image tokens), calculate text_tokens as the remainder
|
| 725 |
+
# 3. If no breakdown at all, assume all completion_tokens are text_tokens
|
| 726 |
+
has_token_breakdown = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0
|
| 727 |
+
if text_tokens == 0:
|
| 728 |
+
if has_token_breakdown:
|
| 729 |
+
# Calculate text tokens as remainder when we have a breakdown
|
| 730 |
+
# This handles cases like OpenAI's reasoning models where text_tokens isn't provided
|
| 731 |
+
text_tokens = max(
|
| 732 |
+
0,
|
| 733 |
+
usage.completion_tokens
|
| 734 |
+
- reasoning_tokens
|
| 735 |
+
- audio_tokens
|
| 736 |
+
- image_tokens,
|
| 737 |
+
)
|
| 738 |
+
else:
|
| 739 |
+
# No breakdown at all, all tokens are text tokens
|
| 740 |
+
text_tokens = usage.completion_tokens
|
| 741 |
+
is_text_tokens_total = True
|
| 742 |
+
## TEXT COST
|
| 743 |
+
completion_cost = float(text_tokens) * completion_base_cost
|
| 744 |
+
|
| 745 |
+
## AUDIO COST
|
| 746 |
+
if not is_text_tokens_total and audio_tokens is not None and audio_tokens > 0:
|
| 747 |
+
_output_cost_per_audio_token = _get_cost_per_unit(
|
| 748 |
+
model_info, "output_cost_per_audio_token", None
|
| 749 |
+
)
|
| 750 |
+
_output_cost_per_audio_token = (
|
| 751 |
+
_output_cost_per_audio_token
|
| 752 |
+
if _output_cost_per_audio_token is not None
|
| 753 |
+
else completion_base_cost
|
| 754 |
+
)
|
| 755 |
+
completion_cost += float(audio_tokens) * _output_cost_per_audio_token
|
| 756 |
+
|
| 757 |
+
## REASONING COST
|
| 758 |
+
if not is_text_tokens_total and reasoning_tokens and reasoning_tokens > 0:
|
| 759 |
+
_output_cost_per_reasoning_token = _get_cost_per_unit(
|
| 760 |
+
model_info, "output_cost_per_reasoning_token", None
|
| 761 |
+
)
|
| 762 |
+
_output_cost_per_reasoning_token = (
|
| 763 |
+
_output_cost_per_reasoning_token
|
| 764 |
+
if _output_cost_per_reasoning_token is not None
|
| 765 |
+
else completion_base_cost
|
| 766 |
+
)
|
| 767 |
+
completion_cost += float(reasoning_tokens) * _output_cost_per_reasoning_token
|
| 768 |
+
|
| 769 |
+
## IMAGE COST
|
| 770 |
+
if not is_text_tokens_total and image_tokens and image_tokens > 0:
|
| 771 |
+
_output_cost_per_image_token = _get_cost_per_unit(
|
| 772 |
+
model_info, "output_cost_per_image_token", None
|
| 773 |
+
)
|
| 774 |
+
_output_cost_per_image_token = (
|
| 775 |
+
_output_cost_per_image_token
|
| 776 |
+
if _output_cost_per_image_token is not None
|
| 777 |
+
else completion_base_cost
|
| 778 |
+
)
|
| 779 |
+
completion_cost += float(image_tokens) * _output_cost_per_image_token
|
| 780 |
+
|
| 781 |
+
return prompt_cost, completion_cost
|
| 782 |
+
|
| 783 |
+
|
| 784 |
+
def calculate_image_response_cost_from_usage(
|
| 785 |
+
model: str,
|
| 786 |
+
image_response: ImageResponse,
|
| 787 |
+
custom_llm_provider: str,
|
| 788 |
+
) -> Optional[float]:
|
| 789 |
+
"""
|
| 790 |
+
Calculate image generation cost from usage metadata when available.
|
| 791 |
+
|
| 792 |
+
Returns:
|
| 793 |
+
Optional[float]: total cost from token usage, or None when usage metadata
|
| 794 |
+
is missing/incomplete and caller should fall back to flat per-image pricing.
|
| 795 |
+
"""
|
| 796 |
+
usage = image_response.usage
|
| 797 |
+
if usage is None:
|
| 798 |
+
return None
|
| 799 |
+
|
| 800 |
+
prompt_tokens = usage.input_tokens
|
| 801 |
+
completion_tokens = usage.output_tokens
|
| 802 |
+
total_tokens = usage.total_tokens
|
| 803 |
+
|
| 804 |
+
if prompt_tokens is None or completion_tokens is None or total_tokens is None:
|
| 805 |
+
return None
|
| 806 |
+
|
| 807 |
+
# ImageResponse may carry a default zeroed usage object even when provider
|
| 808 |
+
# usage metadata is absent. Treat this as missing usage and fall back.
|
| 809 |
+
if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0:
|
| 810 |
+
return None
|
| 811 |
+
|
| 812 |
+
input_tokens_details = getattr(usage, "input_tokens_details", None)
|
| 813 |
+
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
|
| 814 |
+
if input_tokens_details is not None:
|
| 815 |
+
prompt_tokens_details = PromptTokensDetailsWrapper(
|
| 816 |
+
text_tokens=getattr(input_tokens_details, "text_tokens", None),
|
| 817 |
+
image_tokens=getattr(input_tokens_details, "image_tokens", None),
|
| 818 |
+
cached_tokens=0,
|
| 819 |
+
)
|
| 820 |
+
|
| 821 |
+
normalized_usage = Usage(
|
| 822 |
+
prompt_tokens=prompt_tokens,
|
| 823 |
+
completion_tokens=completion_tokens,
|
| 824 |
+
total_tokens=total_tokens,
|
| 825 |
+
prompt_tokens_details=prompt_tokens_details,
|
| 826 |
+
completion_tokens_details=CompletionTokensDetailsWrapper(
|
| 827 |
+
text_tokens=0,
|
| 828 |
+
image_tokens=completion_tokens,
|
| 829 |
+
reasoning_tokens=0,
|
| 830 |
+
audio_tokens=0,
|
| 831 |
+
),
|
| 832 |
+
)
|
| 833 |
+
|
| 834 |
+
prompt_cost, completion_cost = generic_cost_per_token(
|
| 835 |
+
model=model,
|
| 836 |
+
usage=normalized_usage,
|
| 837 |
+
custom_llm_provider=custom_llm_provider,
|
| 838 |
+
)
|
| 839 |
+
return prompt_cost + completion_cost
|
| 840 |
+
|
| 841 |
+
|
| 842 |
+
class CostCalculatorUtils:
|
| 843 |
+
@staticmethod
|
| 844 |
+
def _call_type_has_image_response(call_type: str) -> bool:
|
| 845 |
+
"""
|
| 846 |
+
Returns True if the call type has an image response
|
| 847 |
+
|
| 848 |
+
eg calls that have image response:
|
| 849 |
+
- Image Generation
|
| 850 |
+
- Image Edit
|
| 851 |
+
- Passthrough Image Generation
|
| 852 |
+
"""
|
| 853 |
+
return call_type in _IMAGE_RESPONSE_CALL_TYPES
|
| 854 |
+
|
| 855 |
+
@staticmethod
|
| 856 |
+
def route_image_generation_cost_calculator(
|
| 857 |
+
model: str,
|
| 858 |
+
completion_response: ImageResponse,
|
| 859 |
+
custom_llm_provider: Optional[str] = None,
|
| 860 |
+
quality: Optional[str] = None,
|
| 861 |
+
n: Optional[int] = None,
|
| 862 |
+
size: Optional[str] = None,
|
| 863 |
+
optional_params: Optional[dict] = None,
|
| 864 |
+
call_type: Optional[str] = None,
|
| 865 |
+
) -> float:
|
| 866 |
+
"""
|
| 867 |
+
Route the image generation cost calculator based on the custom_llm_provider
|
| 868 |
+
"""
|
| 869 |
+
from litellm.cost_calculator import default_image_cost_calculator
|
| 870 |
+
from litellm.llms.azure_ai.image_generation.cost_calculator import (
|
| 871 |
+
cost_calculator as azure_ai_image_cost_calculator,
|
| 872 |
+
)
|
| 873 |
+
from litellm.llms.bedrock.image_generation.cost_calculator import (
|
| 874 |
+
cost_calculator as bedrock_image_cost_calculator,
|
| 875 |
+
)
|
| 876 |
+
from litellm.llms.gemini.image_generation.cost_calculator import (
|
| 877 |
+
cost_calculator as gemini_image_cost_calculator,
|
| 878 |
+
)
|
| 879 |
+
from litellm.llms.recraft.cost_calculator import (
|
| 880 |
+
cost_calculator as recraft_image_cost_calculator,
|
| 881 |
+
)
|
| 882 |
+
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
|
| 883 |
+
cost_calculator as vertex_ai_image_cost_calculator,
|
| 884 |
+
)
|
| 885 |
+
|
| 886 |
+
if size is None:
|
| 887 |
+
size = completion_response.size or "1024-x-1024"
|
| 888 |
+
if quality is None:
|
| 889 |
+
quality = completion_response.quality or "standard"
|
| 890 |
+
if n is None:
|
| 891 |
+
n = len(completion_response.data) if completion_response.data else 0
|
| 892 |
+
|
| 893 |
+
if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
|
| 894 |
+
if isinstance(completion_response, ImageResponse):
|
| 895 |
+
return vertex_ai_image_cost_calculator(
|
| 896 |
+
model=model,
|
| 897 |
+
image_response=completion_response,
|
| 898 |
+
)
|
| 899 |
+
elif custom_llm_provider == litellm.LlmProviders.BEDROCK.value:
|
| 900 |
+
if isinstance(completion_response, ImageResponse):
|
| 901 |
+
return bedrock_image_cost_calculator(
|
| 902 |
+
model=model,
|
| 903 |
+
size=size,
|
| 904 |
+
image_response=completion_response,
|
| 905 |
+
optional_params=optional_params,
|
| 906 |
+
)
|
| 907 |
+
raise TypeError(
|
| 908 |
+
"completion_response must be of type ImageResponse for bedrock image cost calculation"
|
| 909 |
+
)
|
| 910 |
+
elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
|
| 911 |
+
from litellm.llms.recraft.cost_calculator import (
|
| 912 |
+
cost_calculator as recraft_image_cost_calculator,
|
| 913 |
+
)
|
| 914 |
+
|
| 915 |
+
return recraft_image_cost_calculator(
|
| 916 |
+
model=model,
|
| 917 |
+
image_response=completion_response,
|
| 918 |
+
)
|
| 919 |
+
elif custom_llm_provider == litellm.LlmProviders.AIML.value:
|
| 920 |
+
from litellm.llms.aiml.image_generation.cost_calculator import (
|
| 921 |
+
cost_calculator as aiml_image_cost_calculator,
|
| 922 |
+
)
|
| 923 |
+
|
| 924 |
+
return aiml_image_cost_calculator(
|
| 925 |
+
model=model,
|
| 926 |
+
image_response=completion_response,
|
| 927 |
+
)
|
| 928 |
+
elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value:
|
| 929 |
+
from litellm.llms.cometapi.image_generation.cost_calculator import (
|
| 930 |
+
cost_calculator as cometapi_image_cost_calculator,
|
| 931 |
+
)
|
| 932 |
+
|
| 933 |
+
return cometapi_image_cost_calculator(
|
| 934 |
+
model=model,
|
| 935 |
+
image_response=completion_response,
|
| 936 |
+
)
|
| 937 |
+
elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
|
| 938 |
+
if call_type in (
|
| 939 |
+
CallTypes.image_edit.value,
|
| 940 |
+
CallTypes.aimage_edit.value,
|
| 941 |
+
):
|
| 942 |
+
from litellm.llms.gemini.image_edit.cost_calculator import (
|
| 943 |
+
cost_calculator as gemini_image_edit_cost_calculator,
|
| 944 |
+
)
|
| 945 |
+
|
| 946 |
+
return gemini_image_edit_cost_calculator(
|
| 947 |
+
model=model,
|
| 948 |
+
image_response=completion_response,
|
| 949 |
+
)
|
| 950 |
+
from litellm.llms.gemini.image_generation.cost_calculator import (
|
| 951 |
+
cost_calculator as gemini_image_cost_calculator,
|
| 952 |
+
)
|
| 953 |
+
|
| 954 |
+
return gemini_image_cost_calculator(
|
| 955 |
+
model=model,
|
| 956 |
+
image_response=completion_response,
|
| 957 |
+
)
|
| 958 |
+
elif custom_llm_provider == litellm.LlmProviders.AZURE_AI.value:
|
| 959 |
+
return azure_ai_image_cost_calculator(
|
| 960 |
+
model=model,
|
| 961 |
+
image_response=completion_response,
|
| 962 |
+
)
|
| 963 |
+
elif custom_llm_provider == litellm.LlmProviders.FAL_AI.value:
|
| 964 |
+
from litellm.llms.fal_ai.cost_calculator import (
|
| 965 |
+
cost_calculator as fal_ai_image_cost_calculator,
|
| 966 |
+
)
|
| 967 |
+
|
| 968 |
+
return fal_ai_image_cost_calculator(
|
| 969 |
+
model=model,
|
| 970 |
+
image_response=completion_response,
|
| 971 |
+
)
|
| 972 |
+
elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
|
| 973 |
+
from litellm.llms.runwayml.cost_calculator import (
|
| 974 |
+
cost_calculator as runwayml_image_cost_calculator,
|
| 975 |
+
)
|
| 976 |
+
|
| 977 |
+
return runwayml_image_cost_calculator(
|
| 978 |
+
model=model,
|
| 979 |
+
image_response=completion_response,
|
| 980 |
+
)
|
| 981 |
+
elif custom_llm_provider == litellm.LlmProviders.OPENAI.value:
|
| 982 |
+
# Check if this is a gpt-image model (token-based pricing)
|
| 983 |
+
model_lower = model.lower()
|
| 984 |
+
if "gpt-image-1" in model_lower:
|
| 985 |
+
from litellm.llms.openai.image_generation.cost_calculator import (
|
| 986 |
+
cost_calculator as openai_gpt_image_cost_calculator,
|
| 987 |
+
)
|
| 988 |
+
|
| 989 |
+
return openai_gpt_image_cost_calculator(
|
| 990 |
+
model=model,
|
| 991 |
+
image_response=completion_response,
|
| 992 |
+
custom_llm_provider=custom_llm_provider,
|
| 993 |
+
)
|
| 994 |
+
# Fall through to default for DALL-E models
|
| 995 |
+
return default_image_cost_calculator(
|
| 996 |
+
model=model,
|
| 997 |
+
quality=quality,
|
| 998 |
+
custom_llm_provider=custom_llm_provider,
|
| 999 |
+
n=n,
|
| 1000 |
+
size=size,
|
| 1001 |
+
optional_params=optional_params,
|
| 1002 |
+
)
|
| 1003 |
+
elif custom_llm_provider == litellm.LlmProviders.AZURE.value:
|
| 1004 |
+
# Check if this is a gpt-image model (token-based pricing)
|
| 1005 |
+
model_lower = model.lower()
|
| 1006 |
+
if "gpt-image-1" in model_lower:
|
| 1007 |
+
from litellm.llms.openai.image_generation.cost_calculator import (
|
| 1008 |
+
cost_calculator as openai_gpt_image_cost_calculator,
|
| 1009 |
+
)
|
| 1010 |
+
|
| 1011 |
+
return openai_gpt_image_cost_calculator(
|
| 1012 |
+
model=model,
|
| 1013 |
+
image_response=completion_response,
|
| 1014 |
+
custom_llm_provider=custom_llm_provider,
|
| 1015 |
+
)
|
| 1016 |
+
# Fall through to default for DALL-E models
|
| 1017 |
+
return default_image_cost_calculator(
|
| 1018 |
+
model=model,
|
| 1019 |
+
quality=quality,
|
| 1020 |
+
custom_llm_provider=custom_llm_provider,
|
| 1021 |
+
n=n,
|
| 1022 |
+
size=size,
|
| 1023 |
+
optional_params=optional_params,
|
| 1024 |
+
)
|
| 1025 |
+
else:
|
| 1026 |
+
return default_image_cost_calculator(
|
| 1027 |
+
model=model,
|
| 1028 |
+
quality=quality,
|
| 1029 |
+
custom_llm_provider=custom_llm_provider,
|
| 1030 |
+
n=n,
|
| 1031 |
+
size=size,
|
| 1032 |
+
optional_params=optional_params,
|
| 1033 |
+
)
|
| 1034 |
+
return 0.0
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_request_utils.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, Optional
|
| 2 |
+
|
| 3 |
+
import litellm
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def _ensure_extra_body_is_safe(extra_body: Optional[Dict]) -> Optional[Dict]:
|
| 7 |
+
"""
|
| 8 |
+
Ensure that the extra_body sent in the request is safe, otherwise users will see this error
|
| 9 |
+
|
| 10 |
+
"Object of type TextPromptClient is not JSON serializable
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
Relevant Issue: https://github.com/BerriAI/litellm/issues/4140
|
| 14 |
+
"""
|
| 15 |
+
if extra_body is None:
|
| 16 |
+
return None
|
| 17 |
+
|
| 18 |
+
if not isinstance(extra_body, dict):
|
| 19 |
+
return extra_body
|
| 20 |
+
|
| 21 |
+
if "metadata" in extra_body and isinstance(extra_body["metadata"], dict):
|
| 22 |
+
if "prompt" in extra_body["metadata"]:
|
| 23 |
+
_prompt = extra_body["metadata"].get("prompt")
|
| 24 |
+
|
| 25 |
+
# users can send Langfuse TextPromptClient objects, so we need to convert them to dicts
|
| 26 |
+
# Langfuse TextPromptClients have .__dict__ attribute
|
| 27 |
+
if _prompt is not None and hasattr(_prompt, "__dict__"):
|
| 28 |
+
extra_body["metadata"]["prompt"] = _prompt.__dict__
|
| 29 |
+
|
| 30 |
+
return extra_body
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
|
| 34 |
+
"""
|
| 35 |
+
Pick the n cheapest chat models from the LLM provider.
|
| 36 |
+
|
| 37 |
+
Args:
|
| 38 |
+
custom_llm_provider (str): The name of the LLM provider.
|
| 39 |
+
n (int): The number of cheapest models to return.
|
| 40 |
+
|
| 41 |
+
Returns:
|
| 42 |
+
list[str]: A list of the n cheapest chat models.
|
| 43 |
+
"""
|
| 44 |
+
if custom_llm_provider not in litellm.models_by_provider:
|
| 45 |
+
return []
|
| 46 |
+
|
| 47 |
+
known_models = litellm.models_by_provider.get(custom_llm_provider, [])
|
| 48 |
+
model_costs = []
|
| 49 |
+
|
| 50 |
+
for model in known_models:
|
| 51 |
+
try:
|
| 52 |
+
model_info = litellm.get_model_info(
|
| 53 |
+
model=model, custom_llm_provider=custom_llm_provider
|
| 54 |
+
)
|
| 55 |
+
except Exception:
|
| 56 |
+
continue
|
| 57 |
+
if model_info.get("mode") != "chat":
|
| 58 |
+
continue
|
| 59 |
+
_cost = model_info.get("input_cost_per_token", 0) + model_info.get(
|
| 60 |
+
"output_cost_per_token", 0
|
| 61 |
+
)
|
| 62 |
+
model_costs.append((model, _cost))
|
| 63 |
+
|
| 64 |
+
# Sort by cost (ascending)
|
| 65 |
+
model_costs.sort(key=lambda x: x[1])
|
| 66 |
+
|
| 67 |
+
# Return the top n cheapest models
|
| 68 |
+
return [model for model, _ in model_costs[:n]]
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def get_proxy_server_request_headers(litellm_params: Optional[dict]) -> dict:
|
| 72 |
+
"""
|
| 73 |
+
Get the `proxy_server_request` headers from the litellm_params.\
|
| 74 |
+
|
| 75 |
+
Use this if you want to access the request headers made to LiteLLM proxy server.
|
| 76 |
+
"""
|
| 77 |
+
if litellm_params is None:
|
| 78 |
+
return {}
|
| 79 |
+
|
| 80 |
+
proxy_request_headers = (
|
| 81 |
+
litellm_params.get("proxy_server_request", {}).get("headers", {}) or {}
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
return proxy_request_headers
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
ADDED
|
@@ -0,0 +1,833 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
import json
|
| 3 |
+
import time
|
| 4 |
+
import traceback
|
| 5 |
+
from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union, cast
|
| 6 |
+
|
| 7 |
+
import litellm
|
| 8 |
+
from litellm._logging import verbose_logger
|
| 9 |
+
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
|
| 10 |
+
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
| 11 |
+
_extract_reasoning_content,
|
| 12 |
+
)
|
| 13 |
+
from litellm.types.llms.databricks import DatabricksTool
|
| 14 |
+
from litellm.types.llms.openai import (
|
| 15 |
+
ChatCompletionThinkingBlock,
|
| 16 |
+
ImageURLListItem,
|
| 17 |
+
OpenAIModerationResponse,
|
| 18 |
+
)
|
| 19 |
+
from litellm.types.utils import (
|
| 20 |
+
ChatCompletionDeltaToolCall,
|
| 21 |
+
ChatCompletionMessageToolCall,
|
| 22 |
+
ChatCompletionRedactedThinkingBlock,
|
| 23 |
+
Choices,
|
| 24 |
+
CompletionTokensDetailsWrapper,
|
| 25 |
+
Delta,
|
| 26 |
+
EmbeddingResponse,
|
| 27 |
+
Function,
|
| 28 |
+
HiddenParams,
|
| 29 |
+
ImageResponse,
|
| 30 |
+
)
|
| 31 |
+
from litellm.types.utils import Logprobs as TextCompletionLogprobs
|
| 32 |
+
from litellm.types.utils import (
|
| 33 |
+
Message,
|
| 34 |
+
ModelResponse,
|
| 35 |
+
ModelResponseStream,
|
| 36 |
+
PromptTokensDetailsWrapper,
|
| 37 |
+
RerankResponse,
|
| 38 |
+
StreamingChoices,
|
| 39 |
+
TextChoices,
|
| 40 |
+
TextCompletionResponse,
|
| 41 |
+
TranscriptionResponse,
|
| 42 |
+
TranscriptionUsageDurationObject,
|
| 43 |
+
TranscriptionUsageTokensObject,
|
| 44 |
+
Usage,
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
from .get_headers import get_response_headers
|
| 48 |
+
|
| 49 |
+
_MESSAGE_FIELDS: frozenset = frozenset(Message.model_fields.keys())
|
| 50 |
+
_CHOICES_FIELDS: frozenset = frozenset(Choices.model_fields.keys())
|
| 51 |
+
_MODEL_RESPONSE_FIELDS: frozenset = frozenset(ModelResponse.model_fields.keys()) | {
|
| 52 |
+
"usage"
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _normalize_images_for_message(
|
| 57 |
+
images: Optional[List[dict]],
|
| 58 |
+
) -> Optional[List[ImageURLListItem]]:
|
| 59 |
+
"""
|
| 60 |
+
Ensure each image has an 'index' field, as required by ImageURLListItem.
|
| 61 |
+
Some providers (e.g. OpenRouter) return images without index.
|
| 62 |
+
"""
|
| 63 |
+
if not images:
|
| 64 |
+
return cast(Optional[List[ImageURLListItem]], images)
|
| 65 |
+
normalized: List[ImageURLListItem] = []
|
| 66 |
+
for i, img in enumerate(images):
|
| 67 |
+
if isinstance(img, dict) and "index" not in img:
|
| 68 |
+
normalized.append(cast(ImageURLListItem, {**img, "index": i}))
|
| 69 |
+
else:
|
| 70 |
+
normalized.append(cast(ImageURLListItem, img))
|
| 71 |
+
return normalized
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _safe_convert_created_field(created_value) -> int:
|
| 75 |
+
"""
|
| 76 |
+
Safely convert a 'created' field value to an integer.
|
| 77 |
+
|
| 78 |
+
Some providers (like SambaNova) return the 'created' field as a float
|
| 79 |
+
(Unix timestamp with fractional seconds), but LiteLLM expects an integer.
|
| 80 |
+
|
| 81 |
+
Args:
|
| 82 |
+
created_value: The value from response_object["created"]
|
| 83 |
+
|
| 84 |
+
Returns:
|
| 85 |
+
int: Unix timestamp as integer
|
| 86 |
+
"""
|
| 87 |
+
if created_value is None:
|
| 88 |
+
return int(time.time())
|
| 89 |
+
elif isinstance(created_value, int):
|
| 90 |
+
return created_value
|
| 91 |
+
elif isinstance(created_value, float):
|
| 92 |
+
return int(created_value)
|
| 93 |
+
else:
|
| 94 |
+
# for strings, etc
|
| 95 |
+
try:
|
| 96 |
+
return int(float(created_value))
|
| 97 |
+
except (ValueError, TypeError):
|
| 98 |
+
# Fallback to current time if conversion fails
|
| 99 |
+
return int(time.time())
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def convert_tool_call_to_json_mode(
|
| 103 |
+
tool_calls: List[ChatCompletionMessageToolCall],
|
| 104 |
+
convert_tool_call_to_json_mode: bool,
|
| 105 |
+
) -> Tuple[Optional[Message], Optional[str]]:
|
| 106 |
+
if _should_convert_tool_call_to_json_mode(
|
| 107 |
+
tool_calls=tool_calls,
|
| 108 |
+
convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
|
| 109 |
+
):
|
| 110 |
+
# to support 'json_schema' logic on older models
|
| 111 |
+
json_mode_content_str: Optional[str] = tool_calls[0]["function"].get(
|
| 112 |
+
"arguments"
|
| 113 |
+
)
|
| 114 |
+
if json_mode_content_str is not None:
|
| 115 |
+
message = litellm.Message(content=json_mode_content_str)
|
| 116 |
+
finish_reason = "stop"
|
| 117 |
+
return message, finish_reason
|
| 118 |
+
return None, None
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
async def convert_to_streaming_response_async(response_object: Optional[dict] = None):
|
| 122 |
+
"""
|
| 123 |
+
Asynchronously converts a response object to a streaming response.
|
| 124 |
+
|
| 125 |
+
Args:
|
| 126 |
+
response_object (Optional[dict]): The response object to be converted. Defaults to None.
|
| 127 |
+
|
| 128 |
+
Raises:
|
| 129 |
+
Exception: If the response object is None.
|
| 130 |
+
|
| 131 |
+
Yields:
|
| 132 |
+
ModelResponse: The converted streaming response object.
|
| 133 |
+
|
| 134 |
+
Returns:
|
| 135 |
+
None
|
| 136 |
+
"""
|
| 137 |
+
if response_object is None:
|
| 138 |
+
raise Exception("Error in response object format")
|
| 139 |
+
|
| 140 |
+
model_response_object = ModelResponseStream()
|
| 141 |
+
|
| 142 |
+
if model_response_object is None:
|
| 143 |
+
raise Exception("Error in response creating model response object")
|
| 144 |
+
|
| 145 |
+
choice_list: List[StreamingChoices] = []
|
| 146 |
+
|
| 147 |
+
for idx, choice in enumerate(response_object["choices"]):
|
| 148 |
+
if (
|
| 149 |
+
choice["message"].get("tool_calls", None) is not None
|
| 150 |
+
and isinstance(choice["message"]["tool_calls"], list)
|
| 151 |
+
and len(choice["message"]["tool_calls"]) > 0
|
| 152 |
+
and isinstance(choice["message"]["tool_calls"][0], dict)
|
| 153 |
+
):
|
| 154 |
+
pydantic_tool_calls = []
|
| 155 |
+
for index, t in enumerate(choice["message"]["tool_calls"]):
|
| 156 |
+
if "index" not in t:
|
| 157 |
+
t["index"] = index
|
| 158 |
+
pydantic_tool_calls.append(ChatCompletionDeltaToolCall(**t))
|
| 159 |
+
choice["message"]["tool_calls"] = pydantic_tool_calls
|
| 160 |
+
delta = Delta(
|
| 161 |
+
content=choice["message"].get("content", None),
|
| 162 |
+
role=choice["message"]["role"],
|
| 163 |
+
function_call=choice["message"].get("function_call", None),
|
| 164 |
+
tool_calls=choice["message"].get("tool_calls", None),
|
| 165 |
+
)
|
| 166 |
+
finish_reason = choice.get("finish_reason", None)
|
| 167 |
+
|
| 168 |
+
if finish_reason is None:
|
| 169 |
+
finish_reason = choice.get("finish_details")
|
| 170 |
+
|
| 171 |
+
logprobs = choice.get("logprobs", None)
|
| 172 |
+
|
| 173 |
+
choice = StreamingChoices(
|
| 174 |
+
finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs
|
| 175 |
+
)
|
| 176 |
+
choice_list.append(choice)
|
| 177 |
+
|
| 178 |
+
model_response_object.choices = choice_list
|
| 179 |
+
|
| 180 |
+
if "usage" in response_object and response_object["usage"] is not None:
|
| 181 |
+
setattr(
|
| 182 |
+
model_response_object,
|
| 183 |
+
"usage",
|
| 184 |
+
Usage(
|
| 185 |
+
completion_tokens=response_object["usage"].get("completion_tokens", 0),
|
| 186 |
+
prompt_tokens=response_object["usage"].get("prompt_tokens", 0),
|
| 187 |
+
total_tokens=response_object["usage"].get("total_tokens", 0),
|
| 188 |
+
),
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
if "id" in response_object:
|
| 192 |
+
model_response_object.id = response_object["id"]
|
| 193 |
+
|
| 194 |
+
if "created" in response_object:
|
| 195 |
+
model_response_object.created = _safe_convert_created_field(
|
| 196 |
+
response_object["created"]
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
if "system_fingerprint" in response_object:
|
| 200 |
+
model_response_object.system_fingerprint = response_object["system_fingerprint"]
|
| 201 |
+
|
| 202 |
+
if "model" in response_object:
|
| 203 |
+
model_response_object.model = response_object["model"]
|
| 204 |
+
|
| 205 |
+
yield model_response_object
|
| 206 |
+
await asyncio.sleep(0)
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def convert_to_streaming_response(response_object: Optional[dict] = None):
|
| 210 |
+
# used for yielding Cache hits when stream == True
|
| 211 |
+
if response_object is None:
|
| 212 |
+
raise Exception("Error in response object format")
|
| 213 |
+
|
| 214 |
+
model_response_object = ModelResponseStream()
|
| 215 |
+
choice_list: List[StreamingChoices] = []
|
| 216 |
+
for idx, choice in enumerate(response_object["choices"]):
|
| 217 |
+
delta = Delta(**choice["message"])
|
| 218 |
+
finish_reason = choice.get("finish_reason", None)
|
| 219 |
+
if finish_reason is None:
|
| 220 |
+
# gpt-4 vision can return 'finish_reason' or 'finish_details'
|
| 221 |
+
finish_reason = choice.get("finish_details")
|
| 222 |
+
logprobs = choice.get("logprobs", None)
|
| 223 |
+
enhancements = choice.get("enhancements", None)
|
| 224 |
+
choice = StreamingChoices(
|
| 225 |
+
finish_reason=finish_reason,
|
| 226 |
+
index=idx,
|
| 227 |
+
delta=delta,
|
| 228 |
+
logprobs=logprobs,
|
| 229 |
+
enhancements=enhancements,
|
| 230 |
+
)
|
| 231 |
+
|
| 232 |
+
choice_list.append(choice)
|
| 233 |
+
model_response_object.choices = choice_list
|
| 234 |
+
|
| 235 |
+
if "usage" in response_object and response_object["usage"] is not None:
|
| 236 |
+
setattr(model_response_object, "usage", Usage())
|
| 237 |
+
model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) # type: ignore
|
| 238 |
+
model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) # type: ignore
|
| 239 |
+
model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0) # type: ignore
|
| 240 |
+
|
| 241 |
+
if "id" in response_object:
|
| 242 |
+
model_response_object.id = response_object["id"]
|
| 243 |
+
|
| 244 |
+
if "created" in response_object:
|
| 245 |
+
model_response_object.created = _safe_convert_created_field(
|
| 246 |
+
response_object["created"]
|
| 247 |
+
)
|
| 248 |
+
|
| 249 |
+
if "system_fingerprint" in response_object:
|
| 250 |
+
model_response_object.system_fingerprint = response_object["system_fingerprint"]
|
| 251 |
+
|
| 252 |
+
if "model" in response_object:
|
| 253 |
+
model_response_object.model = response_object["model"]
|
| 254 |
+
yield model_response_object
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
from collections import defaultdict
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def _handle_invalid_parallel_tool_calls(
|
| 261 |
+
tool_calls: List[ChatCompletionMessageToolCall],
|
| 262 |
+
):
|
| 263 |
+
"""
|
| 264 |
+
Handle hallucinated parallel tool call from openai - https://community.openai.com/t/model-tries-to-call-unknown-function-multi-tool-use-parallel/490653
|
| 265 |
+
|
| 266 |
+
Code modified from: https://github.com/phdowling/openai_multi_tool_use_parallel_patch/blob/main/openai_multi_tool_use_parallel_patch.py
|
| 267 |
+
"""
|
| 268 |
+
|
| 269 |
+
if tool_calls is None:
|
| 270 |
+
return
|
| 271 |
+
try:
|
| 272 |
+
replacements: Dict[int, List[ChatCompletionMessageToolCall]] = defaultdict(list)
|
| 273 |
+
for i, tool_call in enumerate(tool_calls):
|
| 274 |
+
current_function = tool_call.function.name
|
| 275 |
+
function_args = json.loads(tool_call.function.arguments)
|
| 276 |
+
if current_function == "multi_tool_use.parallel":
|
| 277 |
+
verbose_logger.debug(
|
| 278 |
+
"OpenAI did a weird pseudo-multi-tool-use call, fixing call structure.."
|
| 279 |
+
)
|
| 280 |
+
for _fake_i, _fake_tool_use in enumerate(function_args["tool_uses"]):
|
| 281 |
+
_function_args = _fake_tool_use["parameters"]
|
| 282 |
+
_current_function = _fake_tool_use["recipient_name"]
|
| 283 |
+
if _current_function.startswith("functions."):
|
| 284 |
+
_current_function = _current_function[len("functions.") :]
|
| 285 |
+
|
| 286 |
+
fixed_tc = ChatCompletionMessageToolCall(
|
| 287 |
+
id=f"{tool_call.id}_{_fake_i}",
|
| 288 |
+
type="function",
|
| 289 |
+
function=Function(
|
| 290 |
+
name=_current_function, arguments=json.dumps(_function_args)
|
| 291 |
+
),
|
| 292 |
+
)
|
| 293 |
+
replacements[i].append(fixed_tc)
|
| 294 |
+
|
| 295 |
+
shift = 0
|
| 296 |
+
for i, replacement in replacements.items():
|
| 297 |
+
tool_calls[:] = (
|
| 298 |
+
tool_calls[: i + shift] + replacement + tool_calls[i + shift + 1 :]
|
| 299 |
+
)
|
| 300 |
+
shift += len(replacement)
|
| 301 |
+
|
| 302 |
+
return tool_calls
|
| 303 |
+
except json.JSONDecodeError:
|
| 304 |
+
# if there is a JSONDecodeError, return the original tool_calls
|
| 305 |
+
return tool_calls
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
class LiteLLMResponseObjectHandler:
|
| 309 |
+
@staticmethod
|
| 310 |
+
def convert_to_image_response(
|
| 311 |
+
response_object: dict,
|
| 312 |
+
model_response_object: Optional[ImageResponse] = None,
|
| 313 |
+
hidden_params: Optional[dict] = None,
|
| 314 |
+
) -> ImageResponse:
|
| 315 |
+
response_object.update({"hidden_params": hidden_params})
|
| 316 |
+
|
| 317 |
+
# Handle gpt-image-1 usage field with None values
|
| 318 |
+
if "usage" in response_object and response_object["usage"] is not None:
|
| 319 |
+
usage = response_object["usage"]
|
| 320 |
+
# Check if usage fields are None and provide defaults
|
| 321 |
+
if usage.get("input_tokens") is None:
|
| 322 |
+
usage["input_tokens"] = 0
|
| 323 |
+
if usage.get("output_tokens") is None:
|
| 324 |
+
usage["output_tokens"] = 0
|
| 325 |
+
if usage.get("total_tokens") is None:
|
| 326 |
+
usage["total_tokens"] = usage["input_tokens"] + usage["output_tokens"]
|
| 327 |
+
if usage.get("input_tokens_details") is None:
|
| 328 |
+
usage["input_tokens_details"] = {
|
| 329 |
+
"image_tokens": 0,
|
| 330 |
+
"text_tokens": 0,
|
| 331 |
+
}
|
| 332 |
+
|
| 333 |
+
# Map Responses API naming to Chat Completions API naming for cost calculator
|
| 334 |
+
if usage.get("prompt_tokens") is None:
|
| 335 |
+
usage["prompt_tokens"] = usage.get("input_tokens", 0)
|
| 336 |
+
if usage.get("completion_tokens") is None:
|
| 337 |
+
usage["completion_tokens"] = usage.get("output_tokens", 0)
|
| 338 |
+
|
| 339 |
+
# Convert dicts to wrapper objects so getattr() works in cost calculation
|
| 340 |
+
if isinstance(usage.get("input_tokens_details"), dict):
|
| 341 |
+
usage["prompt_tokens_details"] = PromptTokensDetailsWrapper(
|
| 342 |
+
**usage["input_tokens_details"]
|
| 343 |
+
)
|
| 344 |
+
if isinstance(usage.get("output_tokens_details"), dict):
|
| 345 |
+
usage["completion_tokens_details"] = CompletionTokensDetailsWrapper(
|
| 346 |
+
**usage["output_tokens_details"]
|
| 347 |
+
)
|
| 348 |
+
|
| 349 |
+
if model_response_object is None:
|
| 350 |
+
model_response_object = ImageResponse(**response_object)
|
| 351 |
+
return model_response_object
|
| 352 |
+
else:
|
| 353 |
+
model_response_dict = model_response_object.model_dump()
|
| 354 |
+
|
| 355 |
+
model_response_dict.update(response_object)
|
| 356 |
+
model_response_object = ImageResponse(**model_response_dict)
|
| 357 |
+
return model_response_object
|
| 358 |
+
|
| 359 |
+
@staticmethod
|
| 360 |
+
def convert_to_moderation_response(
|
| 361 |
+
response_object: dict,
|
| 362 |
+
) -> OpenAIModerationResponse:
|
| 363 |
+
return OpenAIModerationResponse(**response_object)
|
| 364 |
+
|
| 365 |
+
@staticmethod
|
| 366 |
+
def convert_chat_to_text_completion(
|
| 367 |
+
response: ModelResponse,
|
| 368 |
+
text_completion_response: TextCompletionResponse,
|
| 369 |
+
custom_llm_provider: Optional[str] = None,
|
| 370 |
+
) -> TextCompletionResponse:
|
| 371 |
+
"""
|
| 372 |
+
Converts a chat completion response to a text completion response format.
|
| 373 |
+
|
| 374 |
+
Note: This is used for huggingface. For OpenAI / Azure Text the providers files directly return TextCompletionResponse which we then send to user
|
| 375 |
+
|
| 376 |
+
Args:
|
| 377 |
+
response (ModelResponse): The chat completion response to convert
|
| 378 |
+
|
| 379 |
+
Returns:
|
| 380 |
+
TextCompletionResponse: The converted text completion response
|
| 381 |
+
|
| 382 |
+
Example:
|
| 383 |
+
chat_response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi"}])
|
| 384 |
+
text_response = convert_chat_to_text_completion(chat_response)
|
| 385 |
+
"""
|
| 386 |
+
transformed_logprobs = LiteLLMResponseObjectHandler._convert_provider_response_logprobs_to_text_completion_logprobs(
|
| 387 |
+
response=response,
|
| 388 |
+
custom_llm_provider=custom_llm_provider,
|
| 389 |
+
)
|
| 390 |
+
|
| 391 |
+
text_completion_response["id"] = response.get("id", None)
|
| 392 |
+
text_completion_response["object"] = "text_completion"
|
| 393 |
+
text_completion_response["created"] = response.get("created", None)
|
| 394 |
+
text_completion_response["model"] = response.get("model", None)
|
| 395 |
+
choices_list: List[TextChoices] = []
|
| 396 |
+
|
| 397 |
+
# Convert each choice to TextChoices
|
| 398 |
+
for choice in response["choices"]:
|
| 399 |
+
text_choices = TextChoices()
|
| 400 |
+
text_choices["text"] = choice["message"]["content"]
|
| 401 |
+
text_choices["index"] = choice["index"]
|
| 402 |
+
text_choices["logprobs"] = transformed_logprobs
|
| 403 |
+
text_choices["finish_reason"] = choice["finish_reason"]
|
| 404 |
+
choices_list.append(text_choices)
|
| 405 |
+
|
| 406 |
+
text_completion_response["choices"] = choices_list
|
| 407 |
+
text_completion_response["usage"] = response.get("usage", None)
|
| 408 |
+
text_completion_response._hidden_params = HiddenParams(
|
| 409 |
+
**response._hidden_params
|
| 410 |
+
)
|
| 411 |
+
return text_completion_response
|
| 412 |
+
|
| 413 |
+
@staticmethod
|
| 414 |
+
def _convert_provider_response_logprobs_to_text_completion_logprobs(
|
| 415 |
+
response: ModelResponse,
|
| 416 |
+
custom_llm_provider: Optional[str] = None,
|
| 417 |
+
) -> Optional[TextCompletionLogprobs]:
|
| 418 |
+
"""
|
| 419 |
+
Convert logprobs from provider to OpenAI.Completion() format
|
| 420 |
+
|
| 421 |
+
Only supported for HF TGI models
|
| 422 |
+
"""
|
| 423 |
+
transformed_logprobs: Optional[TextCompletionLogprobs] = None
|
| 424 |
+
|
| 425 |
+
return transformed_logprobs
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
def _should_convert_tool_call_to_json_mode(
|
| 429 |
+
tool_calls: Optional[
|
| 430 |
+
Union[List[ChatCompletionMessageToolCall], List[DatabricksTool]]
|
| 431 |
+
] = None,
|
| 432 |
+
convert_tool_call_to_json_mode: Optional[bool] = None,
|
| 433 |
+
) -> bool:
|
| 434 |
+
"""
|
| 435 |
+
Determine if tool calls should be converted to JSON mode
|
| 436 |
+
"""
|
| 437 |
+
if (
|
| 438 |
+
convert_tool_call_to_json_mode
|
| 439 |
+
and tool_calls is not None
|
| 440 |
+
and len(tool_calls) == 1
|
| 441 |
+
and tool_calls[0]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME
|
| 442 |
+
):
|
| 443 |
+
return True
|
| 444 |
+
return False
|
| 445 |
+
|
| 446 |
+
|
| 447 |
+
def convert_to_model_response_object( # noqa: PLR0915
|
| 448 |
+
response_object: Optional[dict] = None,
|
| 449 |
+
model_response_object: Optional[
|
| 450 |
+
Union[
|
| 451 |
+
ModelResponse,
|
| 452 |
+
EmbeddingResponse,
|
| 453 |
+
ImageResponse,
|
| 454 |
+
TranscriptionResponse,
|
| 455 |
+
RerankResponse,
|
| 456 |
+
]
|
| 457 |
+
] = None,
|
| 458 |
+
response_type: Literal[
|
| 459 |
+
"completion", "embedding", "image_generation", "audio_transcription", "rerank"
|
| 460 |
+
] = "completion",
|
| 461 |
+
stream=False,
|
| 462 |
+
start_time=None,
|
| 463 |
+
end_time=None,
|
| 464 |
+
hidden_params: Optional[dict] = None,
|
| 465 |
+
_response_headers: Optional[dict] = None,
|
| 466 |
+
convert_tool_call_to_json_mode: Optional[
|
| 467 |
+
bool
|
| 468 |
+
] = None, # used for supporting 'json_schema' on older models
|
| 469 |
+
):
|
| 470 |
+
additional_headers = get_response_headers(_response_headers)
|
| 471 |
+
|
| 472 |
+
if hidden_params is None:
|
| 473 |
+
hidden_params = {}
|
| 474 |
+
|
| 475 |
+
# Preserve existing additional_headers if they contain important provider headers
|
| 476 |
+
# For responses API, additional_headers may already be set with LLM provider headers
|
| 477 |
+
existing_additional_headers = hidden_params.get("additional_headers", {})
|
| 478 |
+
if existing_additional_headers and _response_headers is None:
|
| 479 |
+
# Keep existing headers when _response_headers is None (responses API case)
|
| 480 |
+
additional_headers = existing_additional_headers
|
| 481 |
+
else:
|
| 482 |
+
# Merge new headers with existing ones
|
| 483 |
+
if existing_additional_headers:
|
| 484 |
+
additional_headers.update(existing_additional_headers)
|
| 485 |
+
|
| 486 |
+
hidden_params["additional_headers"] = additional_headers
|
| 487 |
+
|
| 488 |
+
### CHECK IF ERROR IN RESPONSE ### - openrouter returns these in the dictionary
|
| 489 |
+
# Some OpenAI-compatible providers (e.g., Apertis) return empty error objects
|
| 490 |
+
# even on success. Only raise if the error contains meaningful data.
|
| 491 |
+
if (
|
| 492 |
+
response_object is not None
|
| 493 |
+
and "error" in response_object
|
| 494 |
+
and response_object["error"] is not None
|
| 495 |
+
):
|
| 496 |
+
error_obj = response_object["error"]
|
| 497 |
+
has_meaningful_error = False
|
| 498 |
+
|
| 499 |
+
if isinstance(error_obj, dict):
|
| 500 |
+
# Check if error dict has non-empty message or non-null code
|
| 501 |
+
error_message = error_obj.get("message", "")
|
| 502 |
+
error_code = error_obj.get("code")
|
| 503 |
+
has_meaningful_error = bool(error_message) or error_code is not None
|
| 504 |
+
elif isinstance(error_obj, str):
|
| 505 |
+
# String error is meaningful if non-empty
|
| 506 |
+
has_meaningful_error = bool(error_obj)
|
| 507 |
+
else:
|
| 508 |
+
# Any other truthy value is considered meaningful
|
| 509 |
+
has_meaningful_error = True
|
| 510 |
+
|
| 511 |
+
if has_meaningful_error:
|
| 512 |
+
error_args = {"status_code": 422, "message": "Error in response object"}
|
| 513 |
+
if isinstance(error_obj, dict):
|
| 514 |
+
if "code" in error_obj:
|
| 515 |
+
error_args["status_code"] = error_obj["code"]
|
| 516 |
+
if "message" in error_obj:
|
| 517 |
+
if isinstance(error_obj["message"], dict):
|
| 518 |
+
message_str = json.dumps(error_obj["message"])
|
| 519 |
+
else:
|
| 520 |
+
message_str = str(error_obj["message"])
|
| 521 |
+
error_args["message"] = message_str
|
| 522 |
+
raised_exception = Exception()
|
| 523 |
+
setattr(raised_exception, "status_code", error_args["status_code"])
|
| 524 |
+
setattr(raised_exception, "message", error_args["message"])
|
| 525 |
+
raise raised_exception
|
| 526 |
+
|
| 527 |
+
try:
|
| 528 |
+
if response_type == "completion" and (
|
| 529 |
+
model_response_object is None
|
| 530 |
+
or isinstance(model_response_object, ModelResponse)
|
| 531 |
+
):
|
| 532 |
+
if response_object is None or model_response_object is None:
|
| 533 |
+
raise Exception("Error in response object format")
|
| 534 |
+
if stream is True:
|
| 535 |
+
# for returning cached responses, we need to yield a generator
|
| 536 |
+
return convert_to_streaming_response(response_object=response_object)
|
| 537 |
+
choice_list: List[Choices] = []
|
| 538 |
+
|
| 539 |
+
assert response_object["choices"] is not None and isinstance(
|
| 540 |
+
response_object["choices"], Iterable
|
| 541 |
+
)
|
| 542 |
+
|
| 543 |
+
for idx, choice in enumerate(response_object["choices"]):
|
| 544 |
+
## HANDLE JSON MODE - anthropic returns single function call]
|
| 545 |
+
tool_calls = choice["message"].get("tool_calls", None)
|
| 546 |
+
if tool_calls is not None:
|
| 547 |
+
_openai_tool_calls = []
|
| 548 |
+
for _tc in tool_calls:
|
| 549 |
+
_openai_tc = ChatCompletionMessageToolCall(**_tc)
|
| 550 |
+
_openai_tool_calls.append(_openai_tc)
|
| 551 |
+
fixed_tool_calls = _handle_invalid_parallel_tool_calls(
|
| 552 |
+
_openai_tool_calls
|
| 553 |
+
)
|
| 554 |
+
|
| 555 |
+
if fixed_tool_calls is not None:
|
| 556 |
+
tool_calls = fixed_tool_calls
|
| 557 |
+
|
| 558 |
+
message: Optional[Message] = None
|
| 559 |
+
finish_reason: Optional[str] = None
|
| 560 |
+
if _should_convert_tool_call_to_json_mode(
|
| 561 |
+
tool_calls=tool_calls,
|
| 562 |
+
convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
|
| 563 |
+
):
|
| 564 |
+
# to support 'json_schema' logic on older models
|
| 565 |
+
json_mode_content_str: Optional[str] = tool_calls[0][
|
| 566 |
+
"function"
|
| 567 |
+
].get("arguments")
|
| 568 |
+
if json_mode_content_str is not None:
|
| 569 |
+
message = litellm.Message(content=json_mode_content_str)
|
| 570 |
+
finish_reason = "stop"
|
| 571 |
+
if message is None:
|
| 572 |
+
# Preserve provider_specific_fields if already present
|
| 573 |
+
# in the response (e.g. from proxy passthrough)
|
| 574 |
+
provider_specific_fields = dict(
|
| 575 |
+
choice["message"].get("provider_specific_fields", None) or {}
|
| 576 |
+
)
|
| 577 |
+
for f in choice["message"].keys() - _MESSAGE_FIELDS:
|
| 578 |
+
provider_specific_fields[f] = choice["message"][f]
|
| 579 |
+
|
| 580 |
+
# Handle reasoning models that display `reasoning_content` within `content`
|
| 581 |
+
reasoning_content, content = _extract_reasoning_content(
|
| 582 |
+
choice["message"]
|
| 583 |
+
)
|
| 584 |
+
|
| 585 |
+
# Handle thinking models that display `thinking_blocks` within `content`
|
| 586 |
+
thinking_blocks: Optional[
|
| 587 |
+
List[
|
| 588 |
+
Union[
|
| 589 |
+
ChatCompletionThinkingBlock,
|
| 590 |
+
ChatCompletionRedactedThinkingBlock,
|
| 591 |
+
]
|
| 592 |
+
]
|
| 593 |
+
] = None
|
| 594 |
+
if "thinking_blocks" in choice["message"]:
|
| 595 |
+
thinking_blocks = choice["message"]["thinking_blocks"]
|
| 596 |
+
provider_specific_fields["thinking_blocks"] = thinking_blocks
|
| 597 |
+
|
| 598 |
+
if reasoning_content:
|
| 599 |
+
provider_specific_fields[
|
| 600 |
+
"reasoning_content"
|
| 601 |
+
] = reasoning_content
|
| 602 |
+
|
| 603 |
+
message = Message(
|
| 604 |
+
content=content,
|
| 605 |
+
role=choice["message"]["role"] or "assistant",
|
| 606 |
+
function_call=choice["message"].get("function_call", None),
|
| 607 |
+
tool_calls=tool_calls,
|
| 608 |
+
audio=choice["message"].get("audio", None),
|
| 609 |
+
provider_specific_fields=provider_specific_fields,
|
| 610 |
+
reasoning_content=reasoning_content,
|
| 611 |
+
thinking_blocks=thinking_blocks,
|
| 612 |
+
annotations=choice["message"].get("annotations", None),
|
| 613 |
+
images=_normalize_images_for_message(
|
| 614 |
+
choice["message"].get("images", None)
|
| 615 |
+
),
|
| 616 |
+
)
|
| 617 |
+
finish_reason = choice.get("finish_reason", None)
|
| 618 |
+
if finish_reason is None:
|
| 619 |
+
# gpt-4 vision can return 'finish_reason' or 'finish_details'
|
| 620 |
+
finish_reason = choice.get("finish_details") or "stop"
|
| 621 |
+
if (
|
| 622 |
+
finish_reason == "stop"
|
| 623 |
+
and message.tool_calls
|
| 624 |
+
and len(message.tool_calls) > 0
|
| 625 |
+
):
|
| 626 |
+
finish_reason = "tool_calls"
|
| 627 |
+
|
| 628 |
+
## PROVIDER SPECIFIC FIELDS ##
|
| 629 |
+
provider_specific_fields = {
|
| 630 |
+
f: choice[f] for f in choice.keys() - _CHOICES_FIELDS
|
| 631 |
+
}
|
| 632 |
+
|
| 633 |
+
logprobs = choice.get("logprobs", None)
|
| 634 |
+
enhancements = choice.get("enhancements", None)
|
| 635 |
+
choice = Choices(
|
| 636 |
+
finish_reason=finish_reason,
|
| 637 |
+
index=idx,
|
| 638 |
+
message=message,
|
| 639 |
+
logprobs=logprobs,
|
| 640 |
+
enhancements=enhancements,
|
| 641 |
+
provider_specific_fields=provider_specific_fields,
|
| 642 |
+
)
|
| 643 |
+
choice_list.append(choice)
|
| 644 |
+
model_response_object.choices = choice_list # type: ignore
|
| 645 |
+
|
| 646 |
+
if "usage" in response_object and response_object["usage"] is not None:
|
| 647 |
+
usage_object = litellm.Usage(**response_object["usage"])
|
| 648 |
+
setattr(model_response_object, "usage", usage_object)
|
| 649 |
+
if "created" in response_object:
|
| 650 |
+
model_response_object.created = _safe_convert_created_field(
|
| 651 |
+
response_object["created"]
|
| 652 |
+
)
|
| 653 |
+
|
| 654 |
+
if "id" in response_object:
|
| 655 |
+
# Preserve the auto-generated id from ModelResponse.__init__
|
| 656 |
+
# when the provider returns a falsy id (None, "")
|
| 657 |
+
model_response_object.id = (
|
| 658 |
+
response_object["id"] or model_response_object.id
|
| 659 |
+
)
|
| 660 |
+
|
| 661 |
+
if "system_fingerprint" in response_object:
|
| 662 |
+
model_response_object.system_fingerprint = response_object[
|
| 663 |
+
"system_fingerprint"
|
| 664 |
+
]
|
| 665 |
+
|
| 666 |
+
if "model" in response_object:
|
| 667 |
+
if model_response_object.model is None:
|
| 668 |
+
model_response_object.model = response_object["model"]
|
| 669 |
+
elif (
|
| 670 |
+
"/" in model_response_object.model
|
| 671 |
+
and response_object["model"] is not None
|
| 672 |
+
):
|
| 673 |
+
openai_compatible_provider = model_response_object.model.split("/")[
|
| 674 |
+
0
|
| 675 |
+
]
|
| 676 |
+
model_response_object.model = (
|
| 677 |
+
openai_compatible_provider + "/" + response_object["model"]
|
| 678 |
+
)
|
| 679 |
+
|
| 680 |
+
if start_time is not None and end_time is not None:
|
| 681 |
+
if isinstance(start_time, type(end_time)):
|
| 682 |
+
model_response_object._response_ms = ( # type: ignore
|
| 683 |
+
end_time - start_time
|
| 684 |
+
).total_seconds() * 1000
|
| 685 |
+
|
| 686 |
+
if hidden_params is not None:
|
| 687 |
+
if model_response_object._hidden_params is None:
|
| 688 |
+
model_response_object._hidden_params = {}
|
| 689 |
+
model_response_object._hidden_params.update(hidden_params)
|
| 690 |
+
|
| 691 |
+
if _response_headers is not None:
|
| 692 |
+
model_response_object._response_headers = _response_headers
|
| 693 |
+
|
| 694 |
+
for k, v in response_object.items():
|
| 695 |
+
if k not in _MODEL_RESPONSE_FIELDS:
|
| 696 |
+
setattr(model_response_object, k, v)
|
| 697 |
+
|
| 698 |
+
return model_response_object
|
| 699 |
+
elif response_type == "embedding" and (
|
| 700 |
+
model_response_object is None
|
| 701 |
+
or isinstance(model_response_object, EmbeddingResponse)
|
| 702 |
+
):
|
| 703 |
+
if response_object is None:
|
| 704 |
+
raise Exception("Error in response object format")
|
| 705 |
+
|
| 706 |
+
if model_response_object is None:
|
| 707 |
+
model_response_object = EmbeddingResponse()
|
| 708 |
+
|
| 709 |
+
if "model" in response_object:
|
| 710 |
+
model_response_object.model = response_object["model"]
|
| 711 |
+
|
| 712 |
+
if "object" in response_object:
|
| 713 |
+
model_response_object.object = response_object["object"]
|
| 714 |
+
|
| 715 |
+
model_response_object.data = response_object["data"]
|
| 716 |
+
|
| 717 |
+
if "usage" in response_object and response_object["usage"] is not None:
|
| 718 |
+
model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) # type: ignore
|
| 719 |
+
model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) # type: ignore
|
| 720 |
+
model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0) # type: ignore
|
| 721 |
+
|
| 722 |
+
if start_time is not None and end_time is not None:
|
| 723 |
+
model_response_object._response_ms = ( # type: ignore
|
| 724 |
+
end_time - start_time
|
| 725 |
+
).total_seconds() * 1000 # return response latency in ms like openai
|
| 726 |
+
|
| 727 |
+
if hidden_params is not None:
|
| 728 |
+
model_response_object._hidden_params = hidden_params
|
| 729 |
+
|
| 730 |
+
if _response_headers is not None:
|
| 731 |
+
model_response_object._response_headers = _response_headers
|
| 732 |
+
|
| 733 |
+
return model_response_object
|
| 734 |
+
elif response_type == "image_generation" and (
|
| 735 |
+
model_response_object is None
|
| 736 |
+
or isinstance(model_response_object, ImageResponse)
|
| 737 |
+
):
|
| 738 |
+
if response_object is None:
|
| 739 |
+
raise Exception("Error in response object format")
|
| 740 |
+
|
| 741 |
+
return LiteLLMResponseObjectHandler.convert_to_image_response(
|
| 742 |
+
response_object=response_object,
|
| 743 |
+
model_response_object=model_response_object,
|
| 744 |
+
hidden_params=hidden_params,
|
| 745 |
+
)
|
| 746 |
+
|
| 747 |
+
elif response_type == "audio_transcription" and (
|
| 748 |
+
model_response_object is None
|
| 749 |
+
or isinstance(model_response_object, TranscriptionResponse)
|
| 750 |
+
):
|
| 751 |
+
if response_object is None:
|
| 752 |
+
raise Exception("Error in response object format")
|
| 753 |
+
|
| 754 |
+
if model_response_object is None:
|
| 755 |
+
model_response_object = TranscriptionResponse()
|
| 756 |
+
|
| 757 |
+
if "text" in response_object:
|
| 758 |
+
model_response_object.text = response_object["text"]
|
| 759 |
+
|
| 760 |
+
optional_keys = ["language", "task", "duration", "words", "segments"]
|
| 761 |
+
for key in optional_keys: # not guaranteed to be in response
|
| 762 |
+
if key in response_object:
|
| 763 |
+
setattr(model_response_object, key, response_object[key])
|
| 764 |
+
|
| 765 |
+
if "usage" in response_object and response_object["usage"] is not None:
|
| 766 |
+
tr_usage_object: Optional[
|
| 767 |
+
Union[
|
| 768 |
+
TranscriptionUsageDurationObject, TranscriptionUsageTokensObject
|
| 769 |
+
]
|
| 770 |
+
] = None
|
| 771 |
+
|
| 772 |
+
if response_object["usage"].get("type", None) == "duration":
|
| 773 |
+
tr_usage_object = TranscriptionUsageDurationObject(
|
| 774 |
+
**response_object["usage"]
|
| 775 |
+
)
|
| 776 |
+
elif response_object["usage"].get("type", None) == "tokens":
|
| 777 |
+
tr_usage_object = TranscriptionUsageTokensObject(
|
| 778 |
+
**response_object["usage"]
|
| 779 |
+
)
|
| 780 |
+
if tr_usage_object is not None:
|
| 781 |
+
setattr(model_response_object, "usage", tr_usage_object)
|
| 782 |
+
|
| 783 |
+
if hidden_params is not None:
|
| 784 |
+
model_response_object._hidden_params = hidden_params
|
| 785 |
+
|
| 786 |
+
# Store internally-calculated duration in _hidden_params for cost
|
| 787 |
+
# tracking without exposing it in the response body. Must be set
|
| 788 |
+
# after hidden_params assignment to avoid being overwritten.
|
| 789 |
+
if "_audio_transcription_duration" in response_object:
|
| 790 |
+
model_response_object._hidden_params[
|
| 791 |
+
"audio_transcription_duration"
|
| 792 |
+
] = response_object["_audio_transcription_duration"]
|
| 793 |
+
|
| 794 |
+
if _response_headers is not None:
|
| 795 |
+
model_response_object._response_headers = _response_headers
|
| 796 |
+
|
| 797 |
+
return model_response_object
|
| 798 |
+
elif response_type == "rerank" and (
|
| 799 |
+
model_response_object is None
|
| 800 |
+
or isinstance(model_response_object, RerankResponse)
|
| 801 |
+
):
|
| 802 |
+
if response_object is None:
|
| 803 |
+
raise Exception("Error in response object format")
|
| 804 |
+
|
| 805 |
+
if model_response_object is None:
|
| 806 |
+
model_response_object = RerankResponse(**response_object)
|
| 807 |
+
return model_response_object
|
| 808 |
+
|
| 809 |
+
if "id" in response_object:
|
| 810 |
+
model_response_object.id = response_object["id"]
|
| 811 |
+
|
| 812 |
+
if "meta" in response_object:
|
| 813 |
+
model_response_object.meta = response_object["meta"]
|
| 814 |
+
|
| 815 |
+
if "results" in response_object:
|
| 816 |
+
model_response_object.results = response_object["results"]
|
| 817 |
+
|
| 818 |
+
return model_response_object
|
| 819 |
+
except Exception:
|
| 820 |
+
received_args = dict(
|
| 821 |
+
response_object=response_object,
|
| 822 |
+
model_response_object=model_response_object,
|
| 823 |
+
response_type=response_type,
|
| 824 |
+
stream=stream,
|
| 825 |
+
start_time=start_time,
|
| 826 |
+
end_time=end_time,
|
| 827 |
+
hidden_params=hidden_params,
|
| 828 |
+
_response_headers=_response_headers,
|
| 829 |
+
convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
|
| 830 |
+
)
|
| 831 |
+
raise Exception(
|
| 832 |
+
f"Invalid response object {traceback.format_exc()}\n\nreceived_args={received_args}"
|
| 833 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_response_utils/get_api_base.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional, Union
|
| 2 |
+
|
| 3 |
+
import litellm
|
| 4 |
+
from litellm import verbose_logger
|
| 5 |
+
|
| 6 |
+
from ...litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
| 7 |
+
from ...types.router import LiteLLM_Params
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def get_api_base(
|
| 11 |
+
model: str, optional_params: Union[dict, LiteLLM_Params]
|
| 12 |
+
) -> Optional[str]:
|
| 13 |
+
"""
|
| 14 |
+
Returns the api base used for calling the model.
|
| 15 |
+
|
| 16 |
+
Parameters:
|
| 17 |
+
- model: str - the model passed to litellm.completion()
|
| 18 |
+
- optional_params - the 'litellm_params' in router.completion *OR* additional params passed to litellm.completion - eg. api_base, api_key, etc. See `LiteLLM_Params` - https://github.com/BerriAI/litellm/blob/f09e6ba98d65e035a79f73bc069145002ceafd36/litellm/router.py#L67
|
| 19 |
+
|
| 20 |
+
Returns:
|
| 21 |
+
- string (api_base) or None
|
| 22 |
+
|
| 23 |
+
Example:
|
| 24 |
+
```
|
| 25 |
+
from litellm import get_api_base
|
| 26 |
+
|
| 27 |
+
get_api_base(model="gemini/gemini-pro")
|
| 28 |
+
```
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
try:
|
| 32 |
+
if isinstance(optional_params, LiteLLM_Params):
|
| 33 |
+
_optional_params = optional_params
|
| 34 |
+
elif "model" in optional_params:
|
| 35 |
+
_optional_params = LiteLLM_Params(**optional_params)
|
| 36 |
+
else: # prevent needing to copy and pop the dict
|
| 37 |
+
_optional_params = LiteLLM_Params(
|
| 38 |
+
model=model, **optional_params
|
| 39 |
+
) # convert to pydantic object
|
| 40 |
+
except Exception:
|
| 41 |
+
return None
|
| 42 |
+
# get llm provider
|
| 43 |
+
|
| 44 |
+
if _optional_params.api_base is not None:
|
| 45 |
+
return _optional_params.api_base
|
| 46 |
+
|
| 47 |
+
if litellm.model_alias_map and model in litellm.model_alias_map:
|
| 48 |
+
model = litellm.model_alias_map[model]
|
| 49 |
+
try:
|
| 50 |
+
(
|
| 51 |
+
model,
|
| 52 |
+
custom_llm_provider,
|
| 53 |
+
dynamic_api_key,
|
| 54 |
+
dynamic_api_base,
|
| 55 |
+
) = get_llm_provider(
|
| 56 |
+
model=model,
|
| 57 |
+
custom_llm_provider=_optional_params.custom_llm_provider,
|
| 58 |
+
api_base=_optional_params.api_base,
|
| 59 |
+
api_key=_optional_params.api_key,
|
| 60 |
+
)
|
| 61 |
+
except Exception as e:
|
| 62 |
+
verbose_logger.debug("Error occurred in getting api base - {}".format(str(e)))
|
| 63 |
+
custom_llm_provider = None
|
| 64 |
+
dynamic_api_base = None
|
| 65 |
+
|
| 66 |
+
if dynamic_api_base is not None:
|
| 67 |
+
return dynamic_api_base
|
| 68 |
+
|
| 69 |
+
stream: bool = getattr(optional_params, "stream", False)
|
| 70 |
+
|
| 71 |
+
if (
|
| 72 |
+
_optional_params.vertex_location is not None
|
| 73 |
+
and _optional_params.vertex_project is not None
|
| 74 |
+
):
|
| 75 |
+
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
| 76 |
+
from litellm.types.llms.vertex_ai import VertexPartnerProvider
|
| 77 |
+
|
| 78 |
+
if "claude" in model:
|
| 79 |
+
_api_base = VertexBase.create_vertex_url(
|
| 80 |
+
vertex_location=_optional_params.vertex_location,
|
| 81 |
+
vertex_project=_optional_params.vertex_project,
|
| 82 |
+
model=model,
|
| 83 |
+
stream=stream,
|
| 84 |
+
partner=VertexPartnerProvider.claude,
|
| 85 |
+
)
|
| 86 |
+
else:
|
| 87 |
+
if stream:
|
| 88 |
+
_api_base = "{}-aiplatform.googleapis.com/v1/projects/{}/locations/{}/publishers/google/models/{}:streamGenerateContent".format(
|
| 89 |
+
_optional_params.vertex_location,
|
| 90 |
+
_optional_params.vertex_project,
|
| 91 |
+
_optional_params.vertex_location,
|
| 92 |
+
model,
|
| 93 |
+
)
|
| 94 |
+
else:
|
| 95 |
+
_api_base = "{}-aiplatform.googleapis.com/v1/projects/{}/locations/{}/publishers/google/models/{}:generateContent".format(
|
| 96 |
+
_optional_params.vertex_location,
|
| 97 |
+
_optional_params.vertex_project,
|
| 98 |
+
_optional_params.vertex_location,
|
| 99 |
+
model,
|
| 100 |
+
)
|
| 101 |
+
return _api_base
|
| 102 |
+
|
| 103 |
+
if custom_llm_provider is None:
|
| 104 |
+
return None
|
| 105 |
+
|
| 106 |
+
if custom_llm_provider == "gemini":
|
| 107 |
+
if stream:
|
| 108 |
+
_api_base = "https://generativelanguage.googleapis.com/v1beta/models/{}:streamGenerateContent".format(
|
| 109 |
+
model
|
| 110 |
+
)
|
| 111 |
+
else:
|
| 112 |
+
_api_base = "https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent".format(
|
| 113 |
+
model
|
| 114 |
+
)
|
| 115 |
+
return _api_base
|
| 116 |
+
elif custom_llm_provider == "openai":
|
| 117 |
+
_api_base = "https://api.openai.com"
|
| 118 |
+
return _api_base
|
| 119 |
+
return None
|