techprotrade commited on
Commit
6541b2c
·
verified ·
1 Parent(s): 5b3de6c

Add loan-advisor-agent (part 2)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/ARCHITECTURE.md +292 -0
  2. loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/handler.py +1065 -0
  3. loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/tools.py +182 -0
  4. loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/transformation.py +373 -0
  5. loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/weights_biases.py +219 -0
  6. loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/__init__.py +68 -0
  7. loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/http_handler.py +697 -0
  8. loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/__init__.py +15 -0
  9. loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/handler.py +155 -0
  10. loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/streaming_iterator.py +286 -0
  11. loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/transformation.py +299 -0
  12. loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/main.py +644 -0
  13. loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/streaming_iterator.py +271 -0
  14. loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/utils.py +87 -0
  15. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/README.md +13 -0
  16. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/api_route_to_call_types.py +69 -0
  17. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/app_crypto.py +33 -0
  18. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/asyncify.py +116 -0
  19. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/audio_utils/audio_health_check.wav +0 -0
  20. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/audio_utils/utils.py +275 -0
  21. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/cached_imports.py +59 -0
  22. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/cli_token_utils.py +58 -0
  23. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/core_helpers.py +437 -0
  24. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/coroutine_checker.py +64 -0
  25. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/credential_accessor.py +35 -0
  26. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/custom_logger_registry.py +184 -0
  27. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/dd_tracing.py +123 -0
  28. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/default_encoding.py +52 -0
  29. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/dot_notation_indexing.py +255 -0
  30. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/duration_parser.py +391 -0
  31. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/env_utils.py +21 -0
  32. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py +0 -0
  33. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/fallback_utils.py +80 -0
  34. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_blog_posts.py +164 -0
  35. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_litellm_params.py +164 -0
  36. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py +958 -0
  37. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_model_cost_map.py +299 -0
  38. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_provider_specific_headers.py +29 -0
  39. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_supported_openai_params.py +320 -0
  40. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/health_check_helpers.py +219 -0
  41. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/health_check_utils.py +28 -0
  42. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +75 -0
  43. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/json_validation_rule.py +133 -0
  44. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/litellm_logging.py +0 -0
  45. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +729 -0
  46. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +38 -0
  47. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/utils.py +1034 -0
  48. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_request_utils.py +84 -0
  49. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +833 -0
  50. loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_response_utils/get_api_base.py +119 -0
loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/ARCHITECTURE.md ADDED
@@ -0,0 +1,292 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # WebSearch Interception Architecture
2
+
3
+ Server-side WebSearch tool execution for models that don't natively support it (e.g., Bedrock/Claude).
4
+
5
+ ## How It Works
6
+
7
+ User makes **ONE** `litellm.messages.acreate()` call → Gets final answer with search results.
8
+ The agentic loop happens transparently on the server.
9
+
10
+ ## LiteLLM Standard Web Search Tool
11
+
12
+ LiteLLM defines a standard web search tool format (`litellm_web_search`) that all native provider tools are converted to. This enables consistent interception across providers.
13
+
14
+ **Standard Tool Definition** (defined in `tools.py`):
15
+ ```python
16
+ {
17
+ "name": "litellm_web_search",
18
+ "description": "Search the web for information...",
19
+ "input_schema": {
20
+ "type": "object",
21
+ "properties": {
22
+ "query": {"type": "string", "description": "The search query"}
23
+ },
24
+ "required": ["query"]
25
+ }
26
+ }
27
+ ```
28
+
29
+ **Tool Name Constant**: `LITELLM_WEB_SEARCH_TOOL_NAME = "litellm_web_search"` (defined in `litellm/constants.py`)
30
+
31
+ ### Supported Tool Formats
32
+
33
+ The interception system automatically detects and handles:
34
+
35
+ | Tool Format | Example | Provider | Detection Method | Future-Proof |
36
+ |-------------|---------|----------|------------------|-------------|
37
+ | **LiteLLM Standard** | `name="litellm_web_search"` | Any | Direct name match | N/A |
38
+ | **Anthropic Native** | `type="web_search_20250305"` | Bedrock, Claude API | Type prefix: `startswith("web_search_")` | ✅ Yes (web_search_2026, etc.) |
39
+ | **Claude Code CLI** | `name="web_search"`, `type="web_search_20250305"` | Claude Code | Name + type check | ✅ Yes (version-agnostic) |
40
+ | **Legacy** | `name="WebSearch"` | Custom | Name match | N/A (backwards compat) |
41
+
42
+ **Future Compatibility**: The `startswith("web_search_")` check in `tools.py` automatically supports future Anthropic web search versions.
43
+
44
+ ### Claude Code CLI Integration
45
+
46
+ Claude Code (Anthropic's official CLI) sends web search requests using Anthropic's native tool format:
47
+
48
+ ```python
49
+ {
50
+ "type": "web_search_20250305",
51
+ "name": "web_search",
52
+ "max_uses": 8
53
+ }
54
+ ```
55
+
56
+ **What Happens:**
57
+ 1. Claude Code sends native `web_search_20250305` tool to LiteLLM proxy
58
+ 2. LiteLLM intercepts and converts to `litellm_web_search` standard format
59
+ 3. Bedrock receives converted tool (NOT native format)
60
+ 4. Model returns `tool_use` block for `litellm_web_search` (not `server_tool_use`)
61
+ 5. LiteLLM's agentic loop intercepts the `tool_use`
62
+ 6. Executes `litellm.asearch()` using configured provider (Perplexity, Tavily, etc.)
63
+ 7. Returns final answer to Claude Code user
64
+
65
+ **Without Interception**: Bedrock would receive native tool → try to execute natively → return `web_search_tool_result_error` with `invalid_tool_input`
66
+
67
+ **With Interception**: LiteLLM converts → Bedrock returns tool_use → LiteLLM executes search → Returns final answer ✅
68
+
69
+ ### Native Tool Conversion
70
+
71
+ Native tools are converted to LiteLLM standard format **before** sending to the provider:
72
+
73
+ 1. **Conversion Point** (`litellm/llms/anthropic/experimental_pass_through/messages/handler.py`):
74
+ - In `anthropic_messages()` function (lines 60-127)
75
+ - Runs BEFORE the API request is made
76
+ - Detects native web search tools using `is_web_search_tool()`
77
+ - Converts to `litellm_web_search` format using `get_litellm_web_search_tool()`
78
+ - Prevents provider from executing search natively (avoids `web_search_tool_result_error`)
79
+
80
+ 2. **Response Detection** (`transformation.py`):
81
+ - Detects `tool_use` blocks with any web search tool name
82
+ - Handles: `litellm_web_search`, `WebSearch`, `web_search`
83
+ - Extracts search queries for execution
84
+
85
+ **Example Conversion**:
86
+ ```python
87
+ # Input (Claude Code's native tool)
88
+ {
89
+ "type": "web_search_20250305",
90
+ "name": "web_search",
91
+ "max_uses": 8
92
+ }
93
+
94
+ # Output (LiteLLM standard)
95
+ {
96
+ "name": "litellm_web_search",
97
+ "description": "Search the web for information...",
98
+ "input_schema": {...}
99
+ }
100
+ ```
101
+
102
+ ---
103
+
104
+ ## Request Flow
105
+
106
+ ### Without Interception (Client-Side)
107
+ User manually handles tool execution:
108
+ 1. User calls `litellm.messages.acreate()` → Gets `tool_use` response
109
+ 2. User executes `litellm.asearch()`
110
+ 3. User calls `litellm.messages.acreate()` again with results
111
+ 4. User gets final answer
112
+
113
+ **Result**: 2 API calls, manual tool execution
114
+
115
+ ### With Interception (Server-Side)
116
+ Server handles tool execution automatically:
117
+
118
+ ```mermaid
119
+ sequenceDiagram
120
+ participant User
121
+ participant Messages as litellm.messages.acreate()
122
+ participant Handler as llm_http_handler.py
123
+ participant Logger as WebSearchInterceptionLogger
124
+ participant Router as proxy_server.llm_router
125
+ participant Search as litellm.asearch()
126
+ participant Provider as Bedrock API
127
+
128
+ User->>Messages: acreate(tools=[WebSearch])
129
+ Messages->>Handler: async_anthropic_messages_handler()
130
+ Handler->>Provider: Request
131
+ Provider-->>Handler: Response (tool_use)
132
+ Handler->>Logger: async_should_run_agentic_loop()
133
+ Logger->>Logger: Detect WebSearch tool_use
134
+ Logger-->>Handler: (True, tools)
135
+ Handler->>Logger: async_run_agentic_loop(tools)
136
+ Logger->>Router: Get search_provider from search_tools
137
+ Router-->>Logger: search_provider
138
+ Logger->>Search: asearch(query, provider)
139
+ Search-->>Logger: Search results
140
+ Logger->>Logger: Build tool_result message
141
+ Logger->>Messages: acreate() with results
142
+ Messages->>Provider: Request with search results
143
+ Provider-->>Messages: Final answer
144
+ Messages-->>Logger: Final response
145
+ Logger-->>Handler: Final response
146
+ Handler-->>User: Final answer (with search results)
147
+ ```
148
+
149
+ **Result**: 1 API call from user, server handles agentic loop
150
+
151
+ ---
152
+
153
+ ## Key Components
154
+
155
+ | Component | File | Purpose |
156
+ |-----------|------|---------|
157
+ | **WebSearchInterceptionLogger** | `handler.py` | CustomLogger that implements agentic loop hooks |
158
+ | **Tool Standardization** | `tools.py` | Standard tool definition, detection, and utilities |
159
+ | **Tool Name Constant** | `constants.py` | `LITELLM_WEB_SEARCH_TOOL_NAME = "litellm_web_search"` |
160
+ | **Tool Conversion** | `anthropic/.../ handler.py` | Converts native tools to LiteLLM standard before API call |
161
+ | **Transformation Logic** | `transformation.py` | Detect tool_use, build tool_result messages, format search responses |
162
+ | **Agentic Loop Hooks** | `integrations/custom_logger.py` | Base hooks: `async_should_run_agentic_loop()`, `async_run_agentic_loop()` |
163
+ | **Hook Orchestration** | `llms/custom_httpx/llm_http_handler.py` | `_call_agentic_completion_hooks()` - calls hooks after response |
164
+ | **Router Search Tools** | `proxy/proxy_server.py` | `llm_router.search_tools` - configured search providers |
165
+ | **Search Endpoints** | `proxy/search_endpoints/endpoints.py` | Router logic for selecting search provider |
166
+
167
+ ---
168
+
169
+ ## Configuration
170
+
171
+ ```python
172
+ from litellm.integrations.websearch_interception import (
173
+ WebSearchInterceptionLogger,
174
+ get_litellm_web_search_tool,
175
+ )
176
+ from litellm.types.utils import LlmProviders
177
+
178
+ # Enable for Bedrock with specific search tool
179
+ litellm.callbacks = [
180
+ WebSearchInterceptionLogger(
181
+ enabled_providers=[LlmProviders.BEDROCK],
182
+ search_tool_name="my-perplexity-tool" # Optional: uses router's first tool if None
183
+ )
184
+ ]
185
+
186
+ # Make request with LiteLLM standard tool (recommended)
187
+ response = await litellm.messages.acreate(
188
+ model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
189
+ messages=[{"role": "user", "content": "What is LiteLLM?"}],
190
+ tools=[get_litellm_web_search_tool()], # LiteLLM standard
191
+ max_tokens=1024,
192
+ stream=True # Auto-converted to non-streaming
193
+ )
194
+
195
+ # OR send native tools - they're auto-converted to LiteLLM standard
196
+ response = await litellm.messages.acreate(
197
+ model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
198
+ messages=[{"role": "user", "content": "What is LiteLLM?"}],
199
+ tools=[{
200
+ "type": "web_search_20250305", # Native Anthropic format
201
+ "name": "web_search",
202
+ "max_uses": 8
203
+ }],
204
+ max_tokens=1024,
205
+ )
206
+ ```
207
+
208
+ ---
209
+
210
+ ## Streaming Support
211
+
212
+ WebSearch interception works transparently with both streaming and non-streaming requests.
213
+
214
+ **How streaming is handled:**
215
+ 1. User makes request with `stream=True` and WebSearch tool
216
+ 2. Before API call, `anthropic_messages()` detects WebSearch + interception enabled
217
+ 3. Converts `stream=True` → `stream=False` internally
218
+ 4. Agentic loop executes with non-streaming responses
219
+ 5. Final response returned to user (non-streaming)
220
+
221
+ **Why this approach:**
222
+ - Server-side agentic loops require consuming full responses to detect tool_use
223
+ - User opts into this behavior by enabling WebSearch interception
224
+ - Provides seamless experience without client changes
225
+
226
+ **Testing:**
227
+ - **Non-streaming**: `test_websearch_interception_e2e.py`
228
+ - **Streaming**: `test_websearch_interception_streaming_e2e.py`
229
+
230
+ ---
231
+
232
+ ## Search Provider Selection
233
+
234
+ 1. If `search_tool_name` specified → Look up in `llm_router.search_tools`
235
+ 2. If not found or None → Use first available search tool
236
+ 3. If no router or no tools → Fallback to `perplexity`
237
+
238
+ Example router config:
239
+ ```yaml
240
+ search_tools:
241
+ - search_tool_name: "my-perplexity-tool"
242
+ litellm_params:
243
+ search_provider: "perplexity"
244
+ - search_tool_name: "my-tavily-tool"
245
+ litellm_params:
246
+ search_provider: "tavily"
247
+ ```
248
+
249
+ ---
250
+
251
+ ## Message Flow
252
+
253
+ ### Initial Request
254
+ ```python
255
+ messages = [{"role": "user", "content": "What is LiteLLM?"}]
256
+ tools = [{"name": "WebSearch", ...}]
257
+ ```
258
+
259
+ ### First API Call (Internal)
260
+ **Response**: `tool_use` with `name="WebSearch"`, `input={"query": "what is litellm"}`
261
+
262
+ ### Server Processing
263
+ 1. Logger detects WebSearch tool_use
264
+ 2. Looks up search provider from router
265
+ 3. Executes `litellm.asearch(query="what is litellm", search_provider="perplexity")`
266
+ 4. Gets results: `"Title: LiteLLM Docs\nURL: docs.litellm.ai\n..."`
267
+
268
+ ### Follow-Up Request (Internal)
269
+ ```python
270
+ messages = [
271
+ {"role": "user", "content": "What is LiteLLM?"},
272
+ {"role": "assistant", "content": [{"type": "tool_use", ...}]},
273
+ {"role": "user", "content": [{"type": "tool_result", "content": "search results..."}]}
274
+ ]
275
+ ```
276
+
277
+ ### User Receives
278
+ ```python
279
+ response.content[0].text
280
+ # "Based on the search results, LiteLLM is a unified interface..."
281
+ ```
282
+
283
+ ---
284
+
285
+ ## Testing
286
+
287
+ **E2E Tests**:
288
+ - `test_websearch_interception_e2e.py` - Non-streaming real API calls to Bedrock
289
+ - `test_websearch_interception_streaming_e2e.py` - Streaming real API calls to Bedrock
290
+
291
+ **Unit Tests**: `test_websearch_interception.py`
292
+ Mocked tests for tool detection, provider filtering, edge cases.
loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/handler.py ADDED
@@ -0,0 +1,1065 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ WebSearch Interception Handler
3
+
4
+ CustomLogger that intercepts WebSearch tool calls for models that don't
5
+ natively support web search (e.g., Bedrock/Claude) and executes them
6
+ server-side using litellm router's search tools.
7
+ """
8
+
9
+ import asyncio
10
+ import math
11
+ import uuid
12
+ from typing import Any, Dict, List, Optional, Tuple, Union, cast
13
+
14
+ import litellm
15
+ from litellm._logging import verbose_logger
16
+ from litellm.anthropic_interface import messages as anthropic_messages
17
+ from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
18
+ from litellm.integrations.custom_logger import CustomLogger
19
+ from litellm.integrations.websearch_interception.tools import (
20
+ get_litellm_web_search_tool,
21
+ get_litellm_web_search_tool_openai,
22
+ is_web_search_tool,
23
+ is_web_search_tool_chat_completion,
24
+ )
25
+ from litellm.integrations.websearch_interception.transformation import (
26
+ WebSearchTransformation,
27
+ )
28
+ from litellm.types.integrations.websearch_interception import (
29
+ WebSearchInterceptionConfig,
30
+ )
31
+ from litellm.types.llms.openai import AllMessageValues
32
+ from litellm.types.utils import LlmProviders
33
+ from litellm.utils import ProviderConfigManager
34
+
35
+
36
+ class WebSearchInterceptionLogger(CustomLogger):
37
+ """
38
+ CustomLogger that intercepts WebSearch tool calls for models that don't
39
+ natively support web search.
40
+
41
+ Implements agentic loop:
42
+ 1. Detects WebSearch tool_use in model response
43
+ 2. Executes litellm.asearch() for each query using router's search tools
44
+ 3. Makes follow-up request with search results
45
+ 4. Returns final response
46
+ """
47
+
48
+ def __init__(
49
+ self,
50
+ enabled_providers: Optional[List[Union[LlmProviders, str]]] = None,
51
+ search_tool_name: Optional[str] = None,
52
+ ):
53
+ """
54
+ Args:
55
+ enabled_providers: List of LLM providers to enable interception for.
56
+ Use LlmProviders enum values (e.g., [LlmProviders.BEDROCK])
57
+ If None or empty list, enables for ALL providers.
58
+ Default: None (all providers enabled)
59
+ search_tool_name: Name of search tool configured in router's search_tools.
60
+ If None, will attempt to use first available search tool.
61
+ """
62
+ super().__init__()
63
+ # Convert enum values to strings for comparison
64
+ if enabled_providers is None:
65
+ self.enabled_providers = [LlmProviders.BEDROCK.value]
66
+ else:
67
+ self.enabled_providers = [
68
+ p.value if isinstance(p, LlmProviders) else p for p in enabled_providers
69
+ ]
70
+ self.search_tool_name = search_tool_name
71
+ self._request_has_websearch = False # Track if current request has web search
72
+
73
+ async def try_short_circuit_search(
74
+ self,
75
+ model: str,
76
+ messages: List[Dict],
77
+ tools: Optional[List[Dict]],
78
+ custom_llm_provider: Optional[str],
79
+ ) -> Optional[Dict[str, Any]]:
80
+ """
81
+ Short-circuit web-search-only requests by executing the search directly.
82
+
83
+ Claude Code sends web search as a separate, standalone /v1/messages
84
+ request with a simple prompt and only web_search tool(s). For providers
85
+ that don't natively support web search (e.g. github_copilot), there is
86
+ no need to route this through the backend LLM — we can detect the
87
+ pattern, execute the search via Tavily/Perplexity, and return a
88
+ synthetic Anthropic response immediately.
89
+
90
+ Args:
91
+ model: Model name from the request
92
+ messages: Messages list from the request
93
+ tools: Tools list from the request
94
+ custom_llm_provider: Provider name
95
+
96
+ Returns:
97
+ An AnthropicMessagesResponse dict if short-circuited, or None to
98
+ continue normal processing.
99
+ """
100
+ if not tools:
101
+ return None
102
+
103
+ # Check if provider is in enabled list
104
+ provider_str = custom_llm_provider or ""
105
+ if (
106
+ self.enabled_providers is not None
107
+ and provider_str not in self.enabled_providers
108
+ ):
109
+ return None
110
+
111
+ # Only short-circuit for providers without native Anthropic Messages
112
+ # support. Providers that have a BaseAnthropicMessagesConfig (bedrock,
113
+ # vertex_ai, azure_ai, anthropic) already use the agentic loop, which
114
+ # includes a follow-up LLM call to synthesize the answer from search
115
+ # results. Short-circuiting those would skip that synthesis step and
116
+ # return raw search text — a regression for existing users.
117
+ try:
118
+ provider_enum = LlmProviders(provider_str)
119
+ anthropic_config = (
120
+ ProviderConfigManager.get_provider_anthropic_messages_config(
121
+ model=model, provider=provider_enum
122
+ )
123
+ )
124
+ if anthropic_config is not None:
125
+ verbose_logger.debug(
126
+ f"WebSearchInterception: Skipping short-circuit for {provider_str} "
127
+ "(provider has native Anthropic Messages support, using agentic loop)"
128
+ )
129
+ return None
130
+ except (ValueError, Exception):
131
+ pass # unknown provider enum → safe to short-circuit
132
+
133
+ # All tools must be web search tools
134
+ if not all(is_web_search_tool(t) for t in tools):
135
+ return None
136
+
137
+ # Extract search query from the last user message
138
+ from litellm.litellm_core_utils.prompt_templates.common_utils import (
139
+ get_last_user_message,
140
+ )
141
+
142
+ query = get_last_user_message(cast(List[AllMessageValues], messages))
143
+ if not query:
144
+ return None
145
+
146
+ verbose_logger.debug(
147
+ "WebSearchInterception: Short-circuit search detected "
148
+ f"(provider={provider_str}, query='{query}')"
149
+ )
150
+
151
+ # Execute search
152
+ try:
153
+ search_result_text = await self._execute_search(query)
154
+ except Exception as e:
155
+ verbose_logger.error(
156
+ f"WebSearchInterception: Short-circuit search failed: {e}"
157
+ )
158
+ search_result_text = f"Search failed: {e}"
159
+
160
+ # Build synthetic Anthropic response
161
+ response: Dict[str, Any] = {
162
+ "id": f"msg_{str(uuid.uuid4())}",
163
+ "type": "message",
164
+ "role": "assistant",
165
+ "model": model,
166
+ "content": [{"type": "text", "text": search_result_text}],
167
+ "stop_reason": "end_turn",
168
+ "stop_sequence": None,
169
+ "usage": {"input_tokens": 0, "output_tokens": 0},
170
+ }
171
+
172
+ verbose_logger.debug(
173
+ "WebSearchInterception: Short-circuit search completed, "
174
+ f"returning synthetic response ({len(search_result_text)} chars)"
175
+ )
176
+ return response
177
+
178
+ async def async_pre_call_deployment_hook(
179
+ self, kwargs: Dict[str, Any], call_type: Optional[Any]
180
+ ) -> Optional[dict]:
181
+ """
182
+ Pre-call hook to convert native Anthropic web_search tools to regular tools.
183
+
184
+ This prevents Bedrock from trying to execute web search server-side (which fails).
185
+ Instead, we convert it to a regular tool so the model returns tool_use blocks
186
+ that we can intercept and execute ourselves.
187
+ """
188
+ # Check if this is for an enabled provider
189
+ # Try top-level kwargs first, then nested litellm_params, then derive from model name
190
+ custom_llm_provider = kwargs.get("custom_llm_provider", "") or kwargs.get(
191
+ "litellm_params", {}
192
+ ).get("custom_llm_provider", "")
193
+ if not custom_llm_provider:
194
+ try:
195
+ _, custom_llm_provider, _, _ = litellm.get_llm_provider(
196
+ model=kwargs.get("model", "")
197
+ )
198
+ except Exception:
199
+ custom_llm_provider = ""
200
+ if custom_llm_provider not in self.enabled_providers:
201
+ return None
202
+
203
+ # Check if request has tools with native web_search
204
+ tools = kwargs.get("tools")
205
+ if not tools:
206
+ return None
207
+
208
+ # Check if any tool is a web search tool (native or already LiteLLM standard)
209
+ has_websearch = any(is_web_search_tool(t) for t in tools)
210
+
211
+ if not has_websearch:
212
+ return None
213
+
214
+ verbose_logger.debug(
215
+ "WebSearchInterception: Converting native web_search tools to LiteLLM standard"
216
+ )
217
+
218
+ # Convert native/custom web_search tools to LiteLLM standard
219
+ converted_tools = []
220
+ for tool in tools:
221
+ if is_web_search_tool(tool):
222
+ # Convert to LiteLLM standard web search tool
223
+ converted_tool = get_litellm_web_search_tool_openai()
224
+ converted_tools.append(converted_tool)
225
+ verbose_logger.debug(
226
+ f"WebSearchInterception: Converted {tool.get('name', 'unknown')} "
227
+ f"(type={tool.get('type', 'none')}) to {LITELLM_WEB_SEARCH_TOOL_NAME}"
228
+ )
229
+ else:
230
+ # Keep other tools as-is
231
+ converted_tools.append(tool)
232
+
233
+ # Update tools in-place and return full kwargs
234
+ kwargs["tools"] = converted_tools
235
+ return kwargs
236
+
237
+ @classmethod
238
+ def from_config_yaml(
239
+ cls, config: WebSearchInterceptionConfig
240
+ ) -> "WebSearchInterceptionLogger":
241
+ """
242
+ Initialize WebSearchInterceptionLogger from proxy config.yaml parameters.
243
+
244
+ Args:
245
+ config: Configuration dictionary from litellm_settings.websearch_interception_params
246
+
247
+ Returns:
248
+ Configured WebSearchInterceptionLogger instance
249
+
250
+ Example:
251
+ From proxy_config.yaml:
252
+ litellm_settings:
253
+ websearch_interception_params:
254
+ enabled_providers: ["bedrock"]
255
+ search_tool_name: "my-perplexity-search"
256
+
257
+ Usage:
258
+ config = litellm_settings.get("websearch_interception_params", {})
259
+ logger = WebSearchInterceptionLogger.from_config_yaml(config)
260
+ """
261
+ # Extract parameters from config
262
+ enabled_providers_str = config.get("enabled_providers", None)
263
+ search_tool_name = config.get("search_tool_name", None)
264
+
265
+ # Convert string provider names to LlmProviders enum values
266
+ enabled_providers: Optional[List[Union[LlmProviders, str]]] = None
267
+ if enabled_providers_str is not None:
268
+ enabled_providers = []
269
+ for provider in enabled_providers_str:
270
+ try:
271
+ # Try to convert string to LlmProviders enum
272
+ provider_enum = LlmProviders(provider)
273
+ enabled_providers.append(provider_enum)
274
+ except ValueError:
275
+ # If conversion fails, keep as string
276
+ enabled_providers.append(provider)
277
+
278
+ return cls(
279
+ enabled_providers=enabled_providers,
280
+ search_tool_name=search_tool_name,
281
+ )
282
+
283
+ async def async_pre_request_hook(
284
+ self, model: str, messages: List[Dict], kwargs: Dict
285
+ ) -> Optional[Dict]:
286
+ """
287
+ Pre-request hook to convert native web search tools to LiteLLM standard.
288
+
289
+ This hook is called before the API request is made, allowing us to:
290
+ 1. Detect native web search tools (web_search_20250305, etc.)
291
+ 2. Convert them to LiteLLM standard format (litellm_web_search)
292
+ 3. Convert stream=True to stream=False for interception
293
+
294
+ This prevents providers like Bedrock from trying to execute web search
295
+ natively (which fails), and ensures our agentic loop can intercept tool_use.
296
+
297
+ Returns:
298
+ Modified kwargs dict with converted tools, or None if no modifications needed
299
+ """
300
+ # Check if this request is for an enabled provider
301
+ custom_llm_provider = kwargs.get("litellm_params", {}).get(
302
+ "custom_llm_provider", ""
303
+ )
304
+
305
+ verbose_logger.debug(
306
+ f"WebSearchInterception: Pre-request hook called"
307
+ f" - custom_llm_provider={custom_llm_provider}"
308
+ f" - enabled_providers={self.enabled_providers or 'ALL'}"
309
+ )
310
+
311
+ if (
312
+ self.enabled_providers is not None
313
+ and custom_llm_provider not in self.enabled_providers
314
+ ):
315
+ verbose_logger.debug(
316
+ f"WebSearchInterception: Skipping - provider {custom_llm_provider} not in {self.enabled_providers}"
317
+ )
318
+ return None
319
+
320
+ # Check if request has tools
321
+ tools = kwargs.get("tools")
322
+ if not tools:
323
+ return None
324
+
325
+ # Check if any tool is a web search tool
326
+ has_websearch = any(is_web_search_tool(t) for t in tools)
327
+ if not has_websearch:
328
+ return None
329
+
330
+ verbose_logger.debug(
331
+ f"WebSearchInterception: Pre-request hook triggered for provider={custom_llm_provider}"
332
+ )
333
+
334
+ # Convert native web search tools to LiteLLM standard
335
+ converted_tools = []
336
+ for tool in tools:
337
+ if is_web_search_tool(tool):
338
+ standard_tool = get_litellm_web_search_tool()
339
+ converted_tools.append(standard_tool)
340
+ verbose_logger.debug(
341
+ f"WebSearchInterception: Converted {tool.get('name', 'unknown')} "
342
+ f"(type={tool.get('type', 'none')}) to {LITELLM_WEB_SEARCH_TOOL_NAME}"
343
+ )
344
+ else:
345
+ converted_tools.append(tool)
346
+
347
+ # Update kwargs with converted tools
348
+ kwargs["tools"] = converted_tools
349
+ verbose_logger.debug(
350
+ f"WebSearchInterception: Tools after conversion: {[t.get('name') for t in converted_tools]}"
351
+ )
352
+
353
+ # Convert stream=True to stream=False for WebSearch interception
354
+ if kwargs.get("stream"):
355
+ verbose_logger.debug(
356
+ "WebSearchInterception: Converting stream=True to stream=False"
357
+ )
358
+ kwargs["stream"] = False
359
+ kwargs["_websearch_interception_converted_stream"] = True
360
+
361
+ return kwargs
362
+
363
+ async def async_should_run_agentic_loop(
364
+ self,
365
+ response: Any,
366
+ model: str,
367
+ messages: List[Dict],
368
+ tools: Optional[List[Dict]],
369
+ stream: bool,
370
+ custom_llm_provider: str,
371
+ kwargs: Dict,
372
+ ) -> Tuple[bool, Dict]:
373
+ """
374
+ Check if WebSearch tool interception is needed for Anthropic Messages API.
375
+
376
+ This is the legacy method for Anthropic-style responses.
377
+ For chat completions, use async_should_run_chat_completion_agentic_loop instead.
378
+ """
379
+
380
+ verbose_logger.debug(
381
+ f"WebSearchInterception: Hook called! provider={custom_llm_provider}, stream={stream}"
382
+ )
383
+ verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}")
384
+
385
+ # Check if provider should be intercepted
386
+ # Note: custom_llm_provider is already normalized by get_llm_provider()
387
+ # (e.g., "bedrock/invoke/..." -> "bedrock")
388
+ if (
389
+ self.enabled_providers is not None
390
+ and custom_llm_provider not in self.enabled_providers
391
+ ):
392
+ verbose_logger.debug(
393
+ f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})"
394
+ )
395
+ return False, {}
396
+
397
+ # Check if tools include any web search tool (LiteLLM standard or native)
398
+ has_websearch_tool = any(is_web_search_tool(t) for t in (tools or []))
399
+ if not has_websearch_tool:
400
+ verbose_logger.debug("WebSearchInterception: No web search tool in request")
401
+ return False, {}
402
+
403
+ # Detect WebSearch tool_use in response (Anthropic format)
404
+ should_intercept, tool_calls = WebSearchTransformation.transform_request(
405
+ response=response,
406
+ stream=stream,
407
+ response_format="anthropic",
408
+ )
409
+
410
+ if not should_intercept:
411
+ verbose_logger.debug(
412
+ "WebSearchInterception: No WebSearch tool_use detected in response"
413
+ )
414
+ return False, {}
415
+
416
+ verbose_logger.debug(
417
+ f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop"
418
+ )
419
+
420
+ # Extract thinking blocks from response content.
421
+ # When extended thinking is enabled, the model response includes
422
+ # thinking/redacted_thinking blocks that must be preserved and
423
+ # prepended to the follow-up assistant message.
424
+ thinking_blocks: List[Dict] = []
425
+ if isinstance(response, dict):
426
+ content = response.get("content", [])
427
+ else:
428
+ content = getattr(response, "content", []) or []
429
+
430
+ for block in content:
431
+ if isinstance(block, dict):
432
+ block_type = block.get("type")
433
+ else:
434
+ block_type = getattr(block, "type", None)
435
+
436
+ if block_type in ("thinking", "redacted_thinking"):
437
+ if isinstance(block, dict):
438
+ thinking_blocks.append(block)
439
+ else:
440
+ # Convert object to dict using getattr, matching the
441
+ # pattern in _detect_from_non_streaming_response
442
+ thinking_block_dict: Dict = {"type": block_type}
443
+ if block_type == "thinking":
444
+ thinking_block_dict["thinking"] = getattr(block, "thinking", "")
445
+ thinking_block_dict["signature"] = getattr(
446
+ block, "signature", ""
447
+ )
448
+ else: # redacted_thinking
449
+ thinking_block_dict["data"] = getattr(block, "data", "")
450
+ thinking_blocks.append(thinking_block_dict)
451
+
452
+ if thinking_blocks:
453
+ verbose_logger.debug(
454
+ f"WebSearchInterception: Extracted {len(thinking_blocks)} thinking block(s) from response"
455
+ )
456
+
457
+ # Return tools dict with tool calls and thinking blocks
458
+ tools_dict = {
459
+ "tool_calls": tool_calls,
460
+ "tool_type": "websearch",
461
+ "provider": custom_llm_provider,
462
+ "response_format": "anthropic",
463
+ "thinking_blocks": thinking_blocks,
464
+ }
465
+ return True, tools_dict
466
+
467
+ async def async_should_run_chat_completion_agentic_loop(
468
+ self,
469
+ response: Any,
470
+ model: str,
471
+ messages: List[Dict],
472
+ tools: Optional[List[Dict]],
473
+ stream: bool,
474
+ custom_llm_provider: str,
475
+ kwargs: Dict,
476
+ ) -> Tuple[bool, Dict]:
477
+ """
478
+ Check if WebSearch tool interception is needed for Chat Completions API.
479
+
480
+ Similar to async_should_run_agentic_loop but for OpenAI-style chat completions.
481
+ """
482
+
483
+ verbose_logger.debug(
484
+ f"WebSearchInterception: Chat completion hook called! provider={custom_llm_provider}, stream={stream}"
485
+ )
486
+ verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}")
487
+
488
+ # Check if provider should be intercepted
489
+ if (
490
+ self.enabled_providers is not None
491
+ and custom_llm_provider not in self.enabled_providers
492
+ ):
493
+ verbose_logger.debug(
494
+ f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})"
495
+ )
496
+ return False, {}
497
+
498
+ # Check if tools include any web search tool (strict check for chat completions)
499
+ has_websearch_tool = any(
500
+ is_web_search_tool_chat_completion(t) for t in (tools or [])
501
+ )
502
+ if not has_websearch_tool:
503
+ verbose_logger.debug(
504
+ "WebSearchInterception: No litellm_web_search tool in request"
505
+ )
506
+ return False, {}
507
+
508
+ # Detect WebSearch tool_calls in response (OpenAI format)
509
+ should_intercept, tool_calls = WebSearchTransformation.transform_request(
510
+ response=response,
511
+ stream=stream,
512
+ response_format="openai",
513
+ )
514
+
515
+ if not should_intercept:
516
+ verbose_logger.debug(
517
+ "WebSearchInterception: No WebSearch tool_calls detected in response"
518
+ )
519
+ return False, {}
520
+
521
+ verbose_logger.debug(
522
+ f"WebSearchInterception: Detected {len(tool_calls)} WebSearch tool call(s), executing agentic loop"
523
+ )
524
+
525
+ # Return tools dict with tool calls
526
+ tools_dict = {
527
+ "tool_calls": tool_calls,
528
+ "tool_type": "websearch",
529
+ "provider": custom_llm_provider,
530
+ "response_format": "openai",
531
+ }
532
+ return True, tools_dict
533
+
534
+ async def async_run_agentic_loop(
535
+ self,
536
+ tools: Dict,
537
+ model: str,
538
+ messages: List[Dict],
539
+ response: Any,
540
+ anthropic_messages_provider_config: Any,
541
+ anthropic_messages_optional_request_params: Dict,
542
+ logging_obj: Any,
543
+ stream: bool,
544
+ kwargs: Dict,
545
+ ) -> Any:
546
+ """
547
+ Execute agentic loop with WebSearch execution for Anthropic Messages API.
548
+
549
+ This is the legacy method for Anthropic-style responses.
550
+ """
551
+
552
+ tool_calls = tools["tool_calls"]
553
+ thinking_blocks = tools.get("thinking_blocks", [])
554
+
555
+ verbose_logger.debug(
556
+ f"WebSearchInterception: Executing agentic loop for {len(tool_calls)} search(es)"
557
+ )
558
+
559
+ return await self._execute_agentic_loop(
560
+ model=model,
561
+ messages=messages,
562
+ tool_calls=tool_calls,
563
+ thinking_blocks=thinking_blocks,
564
+ anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
565
+ logging_obj=logging_obj,
566
+ stream=stream,
567
+ kwargs=kwargs,
568
+ )
569
+
570
+ async def async_run_chat_completion_agentic_loop(
571
+ self,
572
+ tools: Dict,
573
+ model: str,
574
+ messages: List[Dict],
575
+ response: Any,
576
+ optional_params: Dict,
577
+ logging_obj: Any,
578
+ stream: bool,
579
+ kwargs: Dict,
580
+ ) -> Any:
581
+ """
582
+ Execute agentic loop with WebSearch execution for Chat Completions API.
583
+
584
+ Similar to async_run_agentic_loop but for OpenAI-style chat completions.
585
+ """
586
+
587
+ tool_calls = tools["tool_calls"]
588
+ response_format = tools.get("response_format", "openai")
589
+
590
+ verbose_logger.debug(
591
+ f"WebSearchInterception: Executing chat completion agentic loop for {len(tool_calls)} search(es)"
592
+ )
593
+
594
+ return await self._execute_chat_completion_agentic_loop(
595
+ model=model,
596
+ messages=messages,
597
+ tool_calls=tool_calls,
598
+ optional_params=optional_params,
599
+ logging_obj=logging_obj,
600
+ stream=stream,
601
+ kwargs=kwargs,
602
+ response_format=response_format,
603
+ )
604
+
605
+ @staticmethod
606
+ def _resolve_max_tokens(
607
+ optional_params: Dict,
608
+ kwargs: Dict,
609
+ ) -> int:
610
+ """Extract max_tokens and validate against thinking.budget_tokens.
611
+
612
+ Anthropic API requires ``max_tokens > thinking.budget_tokens``.
613
+ If the constraint is violated, auto-adjust to ``budget_tokens + 1024``.
614
+ """
615
+ max_tokens: int = optional_params.get(
616
+ "max_tokens",
617
+ kwargs.get("max_tokens", 1024),
618
+ )
619
+ thinking_param = optional_params.get("thinking")
620
+ if thinking_param and isinstance(thinking_param, dict):
621
+ budget_tokens = thinking_param.get("budget_tokens")
622
+ if (
623
+ budget_tokens is not None
624
+ and isinstance(budget_tokens, (int, float))
625
+ and math.isfinite(budget_tokens)
626
+ and budget_tokens > 0
627
+ ):
628
+ if max_tokens <= budget_tokens:
629
+ adjusted = math.ceil(budget_tokens) + 1024
630
+ verbose_logger.debug(
631
+ "WebSearchInterception: max_tokens=%s <= thinking.budget_tokens=%s, "
632
+ "adjusting to %s to satisfy Anthropic API constraint",
633
+ max_tokens,
634
+ budget_tokens,
635
+ adjusted,
636
+ )
637
+ max_tokens = adjusted
638
+ return max_tokens
639
+
640
+ @staticmethod
641
+ def _prepare_followup_kwargs(kwargs: Dict) -> Dict:
642
+ """Build kwargs for the follow-up call, excluding internal keys.
643
+
644
+ ``litellm_logging_obj`` MUST be excluded so the follow-up call creates
645
+ its own ``Logging`` instance via ``function_setup``. Reusing the
646
+ initial call's logging object triggers the dedup flag
647
+ (``has_logged_async_success``) which silently prevents the initial
648
+ call's spend from being recorded — the root cause of the
649
+ SpendLog / AWS billing mismatch.
650
+ """
651
+ _internal_keys = {"litellm_logging_obj"}
652
+ return {
653
+ k: v
654
+ for k, v in kwargs.items()
655
+ if not k.startswith("_websearch_interception") and k not in _internal_keys
656
+ }
657
+
658
+ async def _execute_agentic_loop(
659
+ self,
660
+ model: str,
661
+ messages: List[Dict],
662
+ tool_calls: List[Dict],
663
+ thinking_blocks: List[Dict],
664
+ anthropic_messages_optional_request_params: Dict,
665
+ logging_obj: Any,
666
+ stream: bool,
667
+ kwargs: Dict,
668
+ ) -> Any:
669
+ """Execute litellm.search() and make follow-up request"""
670
+
671
+ # Extract search queries from tool_use blocks
672
+ search_tasks = []
673
+ for tool_call in tool_calls:
674
+ query = tool_call["input"].get("query")
675
+ if query:
676
+ verbose_logger.debug(
677
+ f"WebSearchInterception: Queuing search for query='{query}'"
678
+ )
679
+ search_tasks.append(self._execute_search(query))
680
+ else:
681
+ verbose_logger.debug(
682
+ f"WebSearchInterception: Tool call {tool_call['id']} has no query"
683
+ )
684
+ # Add empty result for tools without query
685
+ search_tasks.append(self._create_empty_search_result())
686
+
687
+ # Execute searches in parallel
688
+ verbose_logger.debug(
689
+ f"WebSearchInterception: Executing {len(search_tasks)} search(es) in parallel"
690
+ )
691
+ search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
692
+
693
+ # Handle any exceptions in search results
694
+ final_search_results: List[str] = []
695
+ for i, result in enumerate(search_results):
696
+ if isinstance(result, Exception):
697
+ verbose_logger.error(
698
+ f"WebSearchInterception: Search {i} failed with error: {str(result)}"
699
+ )
700
+ final_search_results.append(f"Search failed: {str(result)}")
701
+ elif isinstance(result, str):
702
+ # Explicitly cast to str for type checker
703
+ final_search_results.append(cast(str, result))
704
+ else:
705
+ # Should never happen, but handle for type safety
706
+ verbose_logger.debug(
707
+ f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
708
+ )
709
+ final_search_results.append(str(result))
710
+
711
+ # Build assistant and user messages using transformation
712
+ assistant_message, user_message = WebSearchTransformation.transform_response(
713
+ tool_calls=tool_calls,
714
+ search_results=final_search_results,
715
+ thinking_blocks=thinking_blocks,
716
+ )
717
+
718
+ # Make follow-up request with search results
719
+ # Type cast: user_message is a Dict for Anthropic format (default response_format)
720
+ follow_up_messages = messages + [assistant_message, cast(Dict, user_message)]
721
+
722
+ verbose_logger.debug(
723
+ "WebSearchInterception: Making follow-up request with search results"
724
+ )
725
+ verbose_logger.debug(
726
+ f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}"
727
+ )
728
+ verbose_logger.debug(
729
+ f"WebSearchInterception: Last message (tool_result): {user_message}"
730
+ )
731
+
732
+ # Correlation context for structured logging
733
+ _call_id = getattr(logging_obj, "litellm_call_id", None) or kwargs.get(
734
+ "litellm_call_id", "unknown"
735
+ )
736
+
737
+ full_model_name = model # safe default before try block
738
+
739
+ # Use anthropic_messages.acreate for follow-up request
740
+ try:
741
+ max_tokens = self._resolve_max_tokens(
742
+ anthropic_messages_optional_request_params, kwargs
743
+ )
744
+
745
+ verbose_logger.debug(
746
+ f"WebSearchInterception: Using max_tokens={max_tokens} for follow-up request"
747
+ )
748
+
749
+ # Create a copy of optional params without max_tokens (since we pass it explicitly)
750
+ optional_params_without_max_tokens = {
751
+ k: v
752
+ for k, v in anthropic_messages_optional_request_params.items()
753
+ if k != "max_tokens"
754
+ }
755
+
756
+ kwargs_for_followup = self._prepare_followup_kwargs(kwargs)
757
+
758
+ # Get model from logging_obj.model_call_details["agentic_loop_params"]
759
+ # This preserves the full model name with provider prefix (e.g., "bedrock/invoke/...")
760
+ if logging_obj is not None:
761
+ agentic_params = logging_obj.model_call_details.get(
762
+ "agentic_loop_params", {}
763
+ )
764
+ full_model_name = agentic_params.get("model", model)
765
+ verbose_logger.debug(
766
+ f"WebSearchInterception: Using model name: {full_model_name}"
767
+ )
768
+
769
+ final_response = await anthropic_messages.acreate(
770
+ max_tokens=max_tokens,
771
+ messages=follow_up_messages,
772
+ model=full_model_name,
773
+ **optional_params_without_max_tokens,
774
+ **kwargs_for_followup,
775
+ )
776
+ verbose_logger.debug(
777
+ f"WebSearchInterception: Follow-up request completed, response type: {type(final_response)}"
778
+ )
779
+ verbose_logger.debug(
780
+ f"WebSearchInterception: Final response: {final_response}"
781
+ )
782
+ return final_response
783
+ except Exception as e:
784
+ verbose_logger.exception(
785
+ "WebSearchInterception: Follow-up request failed "
786
+ "[call_id=%s model=%s messages=%d searches=%d]: %s",
787
+ _call_id,
788
+ full_model_name,
789
+ len(follow_up_messages),
790
+ len(final_search_results),
791
+ str(e),
792
+ )
793
+ raise
794
+
795
+ async def _execute_search(self, query: str) -> str:
796
+ """Execute a single web search using router's search tools"""
797
+ try:
798
+ # Import router from proxy_server
799
+ try:
800
+ from litellm.proxy.proxy_server import llm_router
801
+ except ImportError:
802
+ verbose_logger.debug(
803
+ "WebSearchInterception: Could not import llm_router from proxy_server, "
804
+ "falling back to direct litellm.asearch() with perplexity"
805
+ )
806
+ llm_router = None
807
+
808
+ # Determine search provider from router's search_tools
809
+ search_provider: Optional[str] = None
810
+ if llm_router is not None and hasattr(llm_router, "search_tools"):
811
+ if self.search_tool_name:
812
+ # Find specific search tool by name
813
+ matching_tools = [
814
+ tool
815
+ for tool in llm_router.search_tools
816
+ if tool.get("search_tool_name") == self.search_tool_name
817
+ ]
818
+ if matching_tools:
819
+ search_tool = matching_tools[0]
820
+ search_provider = search_tool.get("litellm_params", {}).get(
821
+ "search_provider"
822
+ )
823
+ verbose_logger.debug(
824
+ f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
825
+ f"with provider '{search_provider}'"
826
+ )
827
+ else:
828
+ verbose_logger.debug(
829
+ f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in router, "
830
+ "falling back to first available or perplexity"
831
+ )
832
+
833
+ # If no specific tool or not found, use first available
834
+ if not search_provider and llm_router.search_tools:
835
+ first_tool = llm_router.search_tools[0]
836
+ search_provider = first_tool.get("litellm_params", {}).get(
837
+ "search_provider"
838
+ )
839
+ verbose_logger.debug(
840
+ f"WebSearchInterception: Using first available search tool with provider '{search_provider}'"
841
+ )
842
+
843
+ # Fallback to perplexity if no router or no search tools configured
844
+ if not search_provider:
845
+ search_provider = "perplexity"
846
+ verbose_logger.debug(
847
+ "WebSearchInterception: No search tools configured in router, "
848
+ f"using default provider '{search_provider}'"
849
+ )
850
+
851
+ verbose_logger.debug(
852
+ f"WebSearchInterception: Executing search for '{query}' using provider '{search_provider}'"
853
+ )
854
+ result = await litellm.asearch(query=query, search_provider=search_provider)
855
+
856
+ # Format using transformation function
857
+ search_result_text = WebSearchTransformation.format_search_response(result)
858
+
859
+ verbose_logger.debug(
860
+ f"WebSearchInterception: Search completed for '{query}', got {len(search_result_text)} chars"
861
+ )
862
+ return search_result_text
863
+ except Exception as e:
864
+ verbose_logger.error(
865
+ f"WebSearchInterception: Search failed for '{query}': {str(e)}"
866
+ )
867
+ raise
868
+
869
+ async def _execute_chat_completion_agentic_loop( # noqa: PLR0915
870
+ self,
871
+ model: str,
872
+ messages: List[Dict],
873
+ tool_calls: List[Dict],
874
+ optional_params: Dict,
875
+ logging_obj: Any,
876
+ stream: bool,
877
+ kwargs: Dict,
878
+ response_format: str = "openai",
879
+ ) -> Any:
880
+ """Execute litellm.search() and make follow-up chat completion request"""
881
+
882
+ # Extract search queries from tool_calls
883
+ search_tasks = []
884
+ for tool_call in tool_calls:
885
+ # Handle both Anthropic-style input and OpenAI-style function.arguments
886
+ query = None
887
+ if "input" in tool_call and isinstance(tool_call["input"], dict):
888
+ query = tool_call["input"].get("query")
889
+ elif "function" in tool_call:
890
+ func = tool_call["function"]
891
+ if isinstance(func, dict):
892
+ args = func.get("arguments", {})
893
+ if isinstance(args, dict):
894
+ query = args.get("query")
895
+
896
+ if query:
897
+ verbose_logger.debug(
898
+ f"WebSearchInterception: Queuing search for query='{query}'"
899
+ )
900
+ search_tasks.append(self._execute_search(query))
901
+ else:
902
+ verbose_logger.debug(
903
+ f"WebSearchInterception: Tool call {tool_call.get('id')} has no query"
904
+ )
905
+ # Add empty result for tools without query
906
+ search_tasks.append(self._create_empty_search_result())
907
+
908
+ # Execute searches in parallel
909
+ verbose_logger.debug(
910
+ f"WebSearchInterception: Executing {len(search_tasks)} search(es) in parallel"
911
+ )
912
+ search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
913
+
914
+ # Handle any exceptions in search results
915
+ final_search_results: List[str] = []
916
+ for i, result in enumerate(search_results):
917
+ if isinstance(result, Exception):
918
+ verbose_logger.error(
919
+ f"WebSearchInterception: Search {i} failed with error: {str(result)}"
920
+ )
921
+ final_search_results.append(f"Search failed: {str(result)}")
922
+ elif isinstance(result, str):
923
+ final_search_results.append(cast(str, result))
924
+ else:
925
+ verbose_logger.debug(
926
+ f"WebSearchInterception: Unexpected result type {type(result)} at index {i}"
927
+ )
928
+ final_search_results.append(str(result))
929
+
930
+ # Build assistant and tool messages using transformation
931
+ (
932
+ assistant_message,
933
+ tool_messages_or_user,
934
+ ) = WebSearchTransformation.transform_response(
935
+ tool_calls=tool_calls,
936
+ search_results=final_search_results,
937
+ response_format=response_format,
938
+ )
939
+
940
+ # Make follow-up request with search results
941
+ # For OpenAI format, tool_messages_or_user is a list of tool messages
942
+ if response_format == "openai":
943
+ follow_up_messages = (
944
+ messages + [assistant_message] + cast(List[Dict], tool_messages_or_user)
945
+ )
946
+ else:
947
+ # For Anthropic format (shouldn't happen in this method, but handle it)
948
+ follow_up_messages = messages + [
949
+ assistant_message,
950
+ cast(Dict, tool_messages_or_user),
951
+ ]
952
+
953
+ verbose_logger.debug(
954
+ "WebSearchInterception: Making follow-up chat completion request with search results"
955
+ )
956
+ verbose_logger.debug(
957
+ f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}"
958
+ )
959
+
960
+ # Use litellm.acompletion for follow-up request
961
+ try:
962
+ # Remove internal parameters that shouldn't be passed to follow-up request
963
+ internal_params = {
964
+ "_websearch_interception",
965
+ "acompletion",
966
+ "litellm_logging_obj",
967
+ "custom_llm_provider",
968
+ "model_alias_map",
969
+ "stream_response",
970
+ "custom_prompt_dict",
971
+ }
972
+ kwargs_for_followup = {
973
+ k: v
974
+ for k, v in kwargs.items()
975
+ if not k.startswith("_websearch_interception")
976
+ and k not in internal_params
977
+ }
978
+
979
+ # Get full model name from kwargs
980
+ full_model_name = model
981
+ if "custom_llm_provider" in kwargs:
982
+ custom_llm_provider = kwargs["custom_llm_provider"]
983
+ # Reconstruct full model name with provider prefix if needed
984
+ if not model.startswith(custom_llm_provider):
985
+ # Check if model already has a provider prefix
986
+ if "/" not in model:
987
+ full_model_name = f"{custom_llm_provider}/{model}"
988
+
989
+ verbose_logger.debug(
990
+ f"WebSearchInterception: Using model name: {full_model_name}"
991
+ )
992
+
993
+ # Prepare tools for follow-up request (same as original)
994
+ tools_param = optional_params.get("tools")
995
+
996
+ # Remove tools and extra_body from optional_params to avoid issues
997
+ # extra_body often contains internal LiteLLM params that shouldn't be forwarded
998
+ optional_params_clean = {
999
+ k: v
1000
+ for k, v in optional_params.items()
1001
+ if k
1002
+ not in {
1003
+ "tools",
1004
+ "extra_body",
1005
+ "model_alias_map",
1006
+ "stream_response",
1007
+ "custom_prompt_dict",
1008
+ }
1009
+ }
1010
+
1011
+ final_response = await litellm.acompletion(
1012
+ model=full_model_name,
1013
+ messages=follow_up_messages,
1014
+ tools=tools_param,
1015
+ **optional_params_clean,
1016
+ **kwargs_for_followup,
1017
+ )
1018
+
1019
+ verbose_logger.debug(
1020
+ f"WebSearchInterception: Follow-up request completed, response type: {type(final_response)}"
1021
+ )
1022
+ return final_response
1023
+ except Exception as e:
1024
+ verbose_logger.exception(
1025
+ f"WebSearchInterception: Follow-up request failed: {str(e)}"
1026
+ )
1027
+ raise
1028
+
1029
+ async def _create_empty_search_result(self) -> str:
1030
+ """Create an empty search result for tool calls without queries"""
1031
+ return "No search query provided"
1032
+
1033
+ @staticmethod
1034
+ def initialize_from_proxy_config(
1035
+ litellm_settings: Dict[str, Any],
1036
+ callback_specific_params: Dict[str, Any],
1037
+ ) -> "WebSearchInterceptionLogger":
1038
+ """
1039
+ Static method to initialize WebSearchInterceptionLogger from proxy config.
1040
+
1041
+ Used in callback_utils.py to simplify initialization logic.
1042
+
1043
+ Args:
1044
+ litellm_settings: Dictionary containing litellm_settings from proxy_config.yaml
1045
+ callback_specific_params: Dictionary containing callback-specific parameters
1046
+
1047
+ Returns:
1048
+ Configured WebSearchInterceptionLogger instance
1049
+
1050
+ Example:
1051
+ From callback_utils.py:
1052
+ websearch_obj = WebSearchInterceptionLogger.initialize_from_proxy_config(
1053
+ litellm_settings=litellm_settings,
1054
+ callback_specific_params=callback_specific_params
1055
+ )
1056
+ """
1057
+ # Get websearch_interception_params from litellm_settings or callback_specific_params
1058
+ websearch_params: WebSearchInterceptionConfig = {}
1059
+ if "websearch_interception_params" in litellm_settings:
1060
+ websearch_params = litellm_settings["websearch_interception_params"]
1061
+ elif "websearch_interception" in callback_specific_params:
1062
+ websearch_params = callback_specific_params["websearch_interception"]
1063
+
1064
+ # Use classmethod to initialize from config
1065
+ return WebSearchInterceptionLogger.from_config_yaml(websearch_params)
loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/tools.py ADDED
@@ -0,0 +1,182 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ LiteLLM Web Search Tool Definition
3
+
4
+ This module defines the standard web search tool used across LiteLLM.
5
+ Native provider tools (like Anthropic's web_search_20250305) are converted
6
+ to this format for consistent interception and execution.
7
+ """
8
+
9
+ from typing import Any, Dict
10
+
11
+ from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
12
+
13
+
14
+ def get_litellm_web_search_tool() -> Dict[str, Any]:
15
+ """
16
+ Get the standard LiteLLM web search tool definition.
17
+
18
+ This is the canonical tool definition that all native web search tools
19
+ (like Anthropic's web_search_20250305, Claude Code's web_search, etc.)
20
+ are converted to for interception.
21
+
22
+ Returns:
23
+ Dict containing the Anthropic-style tool definition with:
24
+ - name: Tool name
25
+ - description: What the tool does
26
+ - input_schema: JSON schema for tool parameters
27
+
28
+ Example:
29
+ >>> tool = get_litellm_web_search_tool()
30
+ >>> tool['name']
31
+ 'litellm_web_search'
32
+ """
33
+ return {
34
+ "name": LITELLM_WEB_SEARCH_TOOL_NAME,
35
+ "description": (
36
+ "Search the web for information. Use this when you need current "
37
+ "information or answers to questions that require up-to-date data."
38
+ ),
39
+ "input_schema": {
40
+ "type": "object",
41
+ "properties": {
42
+ "query": {
43
+ "type": "string",
44
+ "description": "The search query to execute",
45
+ }
46
+ },
47
+ "required": ["query"],
48
+ },
49
+ }
50
+
51
+
52
+ def get_litellm_web_search_tool_openai() -> Dict[str, Any]:
53
+ """
54
+ Get the standard LiteLLM web search tool definition in OpenAI format.
55
+
56
+ Used by async_pre_call_deployment_hook which runs in the chat completions
57
+ path where tools must be in OpenAI format (type: "function" with
58
+ function.parameters).
59
+
60
+ Returns:
61
+ Dict containing the OpenAI-style tool definition.
62
+ """
63
+ return {
64
+ "type": "function",
65
+ "function": {
66
+ "name": LITELLM_WEB_SEARCH_TOOL_NAME,
67
+ "description": (
68
+ "Search the web for information. Use this when you need current "
69
+ "information or answers to questions that require up-to-date data."
70
+ ),
71
+ "parameters": {
72
+ "type": "object",
73
+ "properties": {
74
+ "query": {
75
+ "type": "string",
76
+ "description": "The search query to execute",
77
+ }
78
+ },
79
+ "required": ["query"],
80
+ },
81
+ },
82
+ }
83
+
84
+
85
+ def is_web_search_tool_chat_completion(tool: Dict[str, Any]) -> bool:
86
+ """
87
+ Check if a tool is a web search tool for Chat Completions API (strict check).
88
+
89
+ This is a stricter version that ONLY checks for the exact LiteLLM web search tool name.
90
+ Use this for Chat Completions API to avoid false positives with user-defined tools.
91
+
92
+ Detects ONLY:
93
+ - LiteLLM standard: name == "litellm_web_search" (Anthropic format)
94
+ - OpenAI format: type == "function" with function.name == "litellm_web_search"
95
+
96
+ Args:
97
+ tool: Tool dictionary to check
98
+
99
+ Returns:
100
+ True if tool is exactly the LiteLLM web search tool
101
+
102
+ Example:
103
+ >>> is_web_search_tool_chat_completion({"name": "litellm_web_search"})
104
+ True
105
+ >>> is_web_search_tool_chat_completion({"type": "function", "function": {"name": "litellm_web_search"}})
106
+ True
107
+ >>> is_web_search_tool_chat_completion({"name": "web_search"})
108
+ False
109
+ >>> is_web_search_tool_chat_completion({"name": "WebSearch"})
110
+ False
111
+ """
112
+ tool_name = tool.get("name", "")
113
+ tool_type = tool.get("type", "")
114
+
115
+ # Check for OpenAI format: {"type": "function", "function": {"name": "litellm_web_search"}}
116
+ if tool_type == "function" and "function" in tool:
117
+ function_def = tool.get("function", {})
118
+ function_name = function_def.get("name", "")
119
+ if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
120
+ return True
121
+
122
+ # Check for LiteLLM standard tool (Anthropic format)
123
+ if tool_name == LITELLM_WEB_SEARCH_TOOL_NAME:
124
+ return True
125
+
126
+ return False
127
+
128
+
129
+ def is_web_search_tool(tool: Dict[str, Any]) -> bool:
130
+ """
131
+ Check if a tool is a web search tool (native or LiteLLM standard).
132
+
133
+ Detects:
134
+ - LiteLLM standard: name == "litellm_web_search"
135
+ - OpenAI format: type == "function" with function.name == "litellm_web_search"
136
+ - Anthropic native: type starts with "web_search_" (e.g., "web_search_20250305")
137
+ - Claude Code: name == "web_search" with a type field
138
+ - Custom: name == "WebSearch" (legacy format)
139
+
140
+ Args:
141
+ tool: Tool dictionary to check
142
+
143
+ Returns:
144
+ True if tool is a web search tool
145
+
146
+ Example:
147
+ >>> is_web_search_tool({"name": "litellm_web_search"})
148
+ True
149
+ >>> is_web_search_tool({"type": "function", "function": {"name": "litellm_web_search"}})
150
+ True
151
+ >>> is_web_search_tool({"type": "web_search_20250305", "name": "web_search"})
152
+ True
153
+ >>> is_web_search_tool({"name": "calculator"})
154
+ False
155
+ """
156
+ tool_name = tool.get("name", "")
157
+ tool_type = tool.get("type", "")
158
+
159
+ # Check for OpenAI format: {"type": "function", "function": {"name": "..."}}
160
+ if tool_type == "function" and "function" in tool:
161
+ function_def = tool.get("function", {})
162
+ function_name = function_def.get("name", "")
163
+ if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
164
+ return True
165
+
166
+ # Check for LiteLLM standard tool (Anthropic format)
167
+ if tool_name == LITELLM_WEB_SEARCH_TOOL_NAME:
168
+ return True
169
+
170
+ # Check for native Anthropic web_search_* types
171
+ if tool_type.startswith("web_search_"):
172
+ return True
173
+
174
+ # Check for Claude Code's web_search with a type field
175
+ if tool_name == "web_search" and tool_type:
176
+ return True
177
+
178
+ # Check for legacy WebSearch format
179
+ if tool_name == "WebSearch":
180
+ return True
181
+
182
+ return False
loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/websearch_interception/transformation.py ADDED
@@ -0,0 +1,373 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ WebSearch Tool Transformation
3
+
4
+ Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format.
5
+ """
6
+ import json
7
+ from typing import Any, Dict, List, Optional, Tuple, Union
8
+
9
+ from litellm._logging import verbose_logger
10
+ from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
11
+ from litellm.llms.base_llm.search.transformation import SearchResponse
12
+
13
+
14
+ class WebSearchTransformation:
15
+ """
16
+ Transformation class for WebSearch tool interception.
17
+
18
+ Handles transformation between:
19
+ - Anthropic tool_use format → LiteLLM search requests
20
+ - OpenAI tool_calls format → LiteLLM search requests
21
+ - LiteLLM SearchResponse → Anthropic/OpenAI tool_result format
22
+ """
23
+
24
+ @staticmethod
25
+ def transform_request(
26
+ response: Any,
27
+ stream: bool,
28
+ response_format: str = "anthropic",
29
+ ) -> Tuple[bool, List[Dict]]:
30
+ """
31
+ Transform model response to extract WebSearch tool calls.
32
+
33
+ Detects if response contains WebSearch tool_use/tool_calls blocks and extracts
34
+ the search queries for execution.
35
+
36
+ Args:
37
+ response: Model response (dict, AnthropicMessagesResponse, or ModelResponse)
38
+ stream: Whether response is streaming
39
+ response_format: Response format - "anthropic" or "openai" (default: "anthropic")
40
+
41
+ Returns:
42
+ (has_websearch, tool_calls):
43
+ has_websearch: True if WebSearch tool_use found
44
+ tool_calls: List of tool_use/tool_calls dicts with id, name, input/function
45
+
46
+ Note:
47
+ Streaming requests are handled by converting stream=True to stream=False
48
+ in the WebSearchInterceptionLogger.async_log_pre_api_call hook before
49
+ the API request is made. This means by the time this method is called,
50
+ streaming requests have already been converted to non-streaming.
51
+ """
52
+ if stream:
53
+ # This should not happen in practice since we convert streaming to non-streaming
54
+ # in async_log_pre_api_call, but keep this check for safety
55
+ verbose_logger.warning(
56
+ "WebSearchInterception: Unexpected streaming response, skipping interception"
57
+ )
58
+ return False, []
59
+
60
+ # Parse non-streaming response based on format
61
+ if response_format == "openai":
62
+ return WebSearchTransformation._detect_from_openai_response(response)
63
+ else:
64
+ return WebSearchTransformation._detect_from_non_streaming_response(response)
65
+
66
+ @staticmethod
67
+ def _detect_from_non_streaming_response(
68
+ response: Any,
69
+ ) -> Tuple[bool, List[Dict]]:
70
+ """Parse non-streaming response for WebSearch tool_use"""
71
+
72
+ # Handle both dict and object responses
73
+ if isinstance(response, dict):
74
+ content = response.get("content", [])
75
+ else:
76
+ if not hasattr(response, "content"):
77
+ verbose_logger.debug(
78
+ "WebSearchInterception: Response has no content attribute"
79
+ )
80
+ return False, []
81
+ content = response.content or []
82
+
83
+ if not content:
84
+ verbose_logger.debug("WebSearchInterception: Response has empty content")
85
+ return False, []
86
+
87
+ # Find all WebSearch tool_use blocks
88
+ tool_calls = []
89
+ for block in content:
90
+ # Handle both dict and object blocks
91
+ if isinstance(block, dict):
92
+ block_type = block.get("type")
93
+ block_name = block.get("name")
94
+ block_id = block.get("id")
95
+ block_input = block.get("input", {})
96
+ else:
97
+ block_type = getattr(block, "type", None)
98
+ block_name = getattr(block, "name", None)
99
+ block_id = getattr(block, "id", None)
100
+ block_input = getattr(block, "input", {})
101
+
102
+ # Check for LiteLLM standard or legacy web search tools
103
+ # Handles: litellm_web_search, WebSearch, web_search
104
+ if block_type == "tool_use" and block_name in (
105
+ LITELLM_WEB_SEARCH_TOOL_NAME,
106
+ "WebSearch",
107
+ "web_search",
108
+ ):
109
+ # Convert to dict for easier handling
110
+ tool_call = {
111
+ "id": block_id,
112
+ "type": "tool_use",
113
+ "name": block_name, # Preserve original name
114
+ "input": block_input,
115
+ }
116
+ tool_calls.append(tool_call)
117
+ verbose_logger.debug(
118
+ f"WebSearchInterception: Found {block_name} tool_use with id={tool_call['id']}"
119
+ )
120
+
121
+ return len(tool_calls) > 0, tool_calls
122
+
123
+ @staticmethod
124
+ def _detect_from_openai_response(
125
+ response: Any,
126
+ ) -> Tuple[bool, List[Dict]]:
127
+ """Parse OpenAI-style response for WebSearch tool_calls"""
128
+
129
+ # Handle both dict and ModelResponse objects
130
+ if isinstance(response, dict):
131
+ choices = response.get("choices", [])
132
+ else:
133
+ if not hasattr(response, "choices"):
134
+ verbose_logger.debug(
135
+ "WebSearchInterception: Response has no choices attribute"
136
+ )
137
+ return False, []
138
+ choices = response.choices or []
139
+
140
+ if not choices:
141
+ verbose_logger.debug("WebSearchInterception: Response has empty choices")
142
+ return False, []
143
+
144
+ # Get first choice's message
145
+ first_choice = choices[0]
146
+ if isinstance(first_choice, dict):
147
+ message = first_choice.get("message", {})
148
+ else:
149
+ message = getattr(first_choice, "message", None)
150
+
151
+ if not message:
152
+ verbose_logger.debug("WebSearchInterception: First choice has no message")
153
+ return False, []
154
+
155
+ # Get tool_calls from message
156
+ if isinstance(message, dict):
157
+ openai_tool_calls = message.get("tool_calls", [])
158
+ else:
159
+ openai_tool_calls = getattr(message, "tool_calls", None) or []
160
+
161
+ if not openai_tool_calls:
162
+ verbose_logger.debug("WebSearchInterception: Message has no tool_calls")
163
+ return False, []
164
+
165
+ # Find all WebSearch tool calls
166
+ tool_calls = []
167
+ for tool_call in openai_tool_calls:
168
+ # Handle both dict and object tool calls
169
+ if isinstance(tool_call, dict):
170
+ tool_id = tool_call.get("id")
171
+ tool_type = tool_call.get("type")
172
+ function = tool_call.get("function", {})
173
+ function_name = (
174
+ function.get("name")
175
+ if isinstance(function, dict)
176
+ else getattr(function, "name", None)
177
+ )
178
+ function_arguments = (
179
+ function.get("arguments")
180
+ if isinstance(function, dict)
181
+ else getattr(function, "arguments", None)
182
+ )
183
+ else:
184
+ tool_id = getattr(tool_call, "id", None)
185
+ tool_type = getattr(tool_call, "type", None)
186
+ function = getattr(tool_call, "function", None)
187
+ function_name = getattr(function, "name", None) if function else None
188
+ function_arguments = (
189
+ getattr(function, "arguments", None) if function else None
190
+ )
191
+
192
+ # Check for LiteLLM standard or legacy web search tools
193
+ if tool_type == "function" and function_name in (
194
+ LITELLM_WEB_SEARCH_TOOL_NAME,
195
+ "WebSearch",
196
+ "web_search",
197
+ ):
198
+ # Parse arguments (might be JSON string)
199
+ if isinstance(function_arguments, str):
200
+ try:
201
+ arguments = json.loads(function_arguments)
202
+ except json.JSONDecodeError:
203
+ verbose_logger.warning(
204
+ f"WebSearchInterception: Failed to parse function arguments: {function_arguments}"
205
+ )
206
+ arguments = {}
207
+ else:
208
+ arguments = function_arguments or {}
209
+
210
+ # Convert to internal format (similar to Anthropic)
211
+ tool_call_dict = {
212
+ "id": tool_id,
213
+ "type": "function",
214
+ "name": function_name,
215
+ "function": {
216
+ "name": function_name,
217
+ "arguments": arguments,
218
+ },
219
+ "input": arguments, # For compatibility with Anthropic format
220
+ }
221
+ tool_calls.append(tool_call_dict)
222
+ verbose_logger.debug(
223
+ f"WebSearchInterception: Found {function_name} tool_call with id={tool_id}"
224
+ )
225
+
226
+ return len(tool_calls) > 0, tool_calls
227
+
228
+ @staticmethod
229
+ def transform_response(
230
+ tool_calls: List[Dict],
231
+ search_results: List[str],
232
+ response_format: str = "anthropic",
233
+ thinking_blocks: Optional[List[Dict]] = None,
234
+ ) -> Tuple[Dict, Union[Dict, List[Dict]]]:
235
+ """
236
+ Transform LiteLLM search results to Anthropic/OpenAI tool_result format.
237
+
238
+ Builds the assistant and user/tool messages needed for the agentic loop
239
+ follow-up request.
240
+
241
+ Args:
242
+ tool_calls: List of tool_use/tool_calls dicts from transform_request
243
+ search_results: List of search result strings (one per tool_call)
244
+ response_format: Response format - "anthropic" or "openai" (default: "anthropic")
245
+ thinking_blocks: Optional list of thinking/redacted_thinking blocks
246
+ from the model's response. When present, prepended to the
247
+ assistant message content (required by Anthropic API when
248
+ thinking is enabled).
249
+
250
+ Returns:
251
+ (assistant_message, user_or_tool_messages):
252
+ For Anthropic: assistant_message with tool_use blocks, user_message with tool_result blocks
253
+ For OpenAI: assistant_message with tool_calls, tool_messages list with tool results
254
+ """
255
+ if response_format == "openai":
256
+ return WebSearchTransformation._transform_response_openai(
257
+ tool_calls, search_results
258
+ )
259
+ else:
260
+ return WebSearchTransformation._transform_response_anthropic(
261
+ tool_calls, search_results, thinking_blocks=thinking_blocks
262
+ )
263
+
264
+ @staticmethod
265
+ def _transform_response_anthropic(
266
+ tool_calls: List[Dict],
267
+ search_results: List[str],
268
+ thinking_blocks: Optional[List[Dict]] = None,
269
+ ) -> Tuple[Dict, Dict]:
270
+ """Transform to Anthropic format (single user message with tool_result blocks)"""
271
+ # Build assistant message content
272
+ assistant_content: List[Dict] = []
273
+
274
+ # Prepend thinking blocks if present.
275
+ # When extended thinking is enabled, Anthropic requires the assistant
276
+ # message to start with thinking/redacted_thinking blocks before any
277
+ # tool_use blocks. Same pattern as anthropic_messages_pt in factory.py.
278
+ if thinking_blocks:
279
+ assistant_content.extend(thinking_blocks)
280
+
281
+ # Add tool_use blocks
282
+ assistant_content.extend(
283
+ [
284
+ {
285
+ "type": "tool_use",
286
+ "id": tc["id"],
287
+ "name": tc["name"],
288
+ "input": tc["input"],
289
+ }
290
+ for tc in tool_calls
291
+ ]
292
+ )
293
+
294
+ assistant_message = {
295
+ "role": "assistant",
296
+ "content": assistant_content,
297
+ }
298
+
299
+ # Build user message with tool_result blocks
300
+ user_message = {
301
+ "role": "user",
302
+ "content": [
303
+ {
304
+ "type": "tool_result",
305
+ "tool_use_id": tool_calls[i]["id"],
306
+ "content": search_results[i],
307
+ }
308
+ for i in range(len(tool_calls))
309
+ ],
310
+ }
311
+
312
+ return assistant_message, user_message
313
+
314
+ @staticmethod
315
+ def _transform_response_openai(
316
+ tool_calls: List[Dict],
317
+ search_results: List[str],
318
+ ) -> Tuple[Dict, List[Dict]]:
319
+ """Transform to OpenAI format (assistant with tool_calls, separate tool messages)"""
320
+ # Build assistant message with tool_calls
321
+ assistant_message = {
322
+ "role": "assistant",
323
+ "tool_calls": [
324
+ {
325
+ "id": tc["id"],
326
+ "type": "function",
327
+ "function": {
328
+ "name": tc["name"],
329
+ "arguments": json.dumps(tc["input"])
330
+ if isinstance(tc["input"], dict)
331
+ else str(tc["input"]),
332
+ },
333
+ }
334
+ for tc in tool_calls
335
+ ],
336
+ }
337
+
338
+ # Build separate tool messages (one per tool call)
339
+ tool_messages = [
340
+ {
341
+ "role": "tool",
342
+ "tool_call_id": tool_calls[i]["id"],
343
+ "content": search_results[i],
344
+ }
345
+ for i in range(len(tool_calls))
346
+ ]
347
+
348
+ return assistant_message, tool_messages
349
+
350
+ @staticmethod
351
+ def format_search_response(result: SearchResponse) -> str:
352
+ """
353
+ Format SearchResponse as text for tool_result content.
354
+
355
+ Args:
356
+ result: SearchResponse from litellm.asearch()
357
+
358
+ Returns:
359
+ Formatted text with Title, URL, Snippet for each result
360
+ """
361
+ # Convert SearchResponse to string
362
+ if hasattr(result, "results") and result.results:
363
+ # Format results as text
364
+ search_result_text = "\n\n".join(
365
+ [
366
+ f"Title: {r.title}\nURL: {r.url}\nSnippet: {r.snippet}"
367
+ for r in result.results
368
+ ]
369
+ )
370
+ else:
371
+ search_result_text = str(result)
372
+
373
+ return search_result_text
loan-advisor-agent/.venv/Lib/site-packages/litellm/integrations/weights_biases.py ADDED
@@ -0,0 +1,219 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ imported_openAIResponse = True
2
+ try:
3
+ import io
4
+ import logging
5
+ import sys
6
+ from typing import Any, Dict, List, Optional, TypeVar
7
+
8
+ from wandb.sdk.data_types import trace_tree
9
+
10
+ if sys.version_info >= (3, 8):
11
+ from typing import Literal, Protocol
12
+ else:
13
+ from typing_extensions import Literal, Protocol
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ K = TypeVar("K", bound=str)
18
+ V = TypeVar("V")
19
+
20
+ class OpenAIResponse(Protocol[K, V]): # type: ignore
21
+ # contains a (known) object attribute
22
+ object: Literal["chat.completion", "edit", "text_completion"]
23
+
24
+ def __getitem__(self, key: K) -> V:
25
+ ... # noqa
26
+
27
+ def get(self, key: K, default: Optional[V] = None) -> Optional[V]: # noqa
28
+ ... # pragma: no cover
29
+
30
+ class OpenAIRequestResponseResolver:
31
+ def __call__(
32
+ self,
33
+ request: Dict[str, Any],
34
+ response: OpenAIResponse,
35
+ time_elapsed: float,
36
+ ) -> Optional[trace_tree.WBTraceTree]:
37
+ try:
38
+ if response["object"] == "edit":
39
+ return self._resolve_edit(request, response, time_elapsed)
40
+ elif response["object"] == "text_completion":
41
+ return self._resolve_completion(request, response, time_elapsed)
42
+ elif response["object"] == "chat.completion":
43
+ return self._resolve_chat_completion(
44
+ request, response, time_elapsed
45
+ )
46
+ else:
47
+ logger.debug(
48
+ f"Unknown OpenAI response object: {response['object']}"
49
+ )
50
+ except Exception as e:
51
+ logger.warning(f"Failed to resolve request/response: {e}")
52
+ return None
53
+
54
+ @staticmethod
55
+ def results_to_trace_tree(
56
+ request: Dict[str, Any],
57
+ response: OpenAIResponse,
58
+ results: List[trace_tree.Result],
59
+ time_elapsed: float,
60
+ ) -> trace_tree.WBTraceTree:
61
+ """Converts the request, response, and results into a trace tree.
62
+
63
+ params:
64
+ request: The request dictionary
65
+ response: The response object
66
+ results: A list of results object
67
+ time_elapsed: The time elapsed in seconds
68
+ returns:
69
+ A wandb trace tree object.
70
+ """
71
+ start_time_ms = int(round(response["created"] * 1000))
72
+ end_time_ms = start_time_ms + int(round(time_elapsed * 1000))
73
+ span = trace_tree.Span(
74
+ name=f"{response.get('model', 'openai')}_{response['object']}_{response.get('created')}",
75
+ attributes=dict(response), # type: ignore
76
+ start_time_ms=start_time_ms,
77
+ end_time_ms=end_time_ms,
78
+ span_kind=trace_tree.SpanKind.LLM,
79
+ results=results,
80
+ )
81
+ model_obj = {"request": request, "response": response, "_kind": "openai"}
82
+ return trace_tree.WBTraceTree(root_span=span, model_dict=model_obj)
83
+
84
+ def _resolve_edit(
85
+ self,
86
+ request: Dict[str, Any],
87
+ response: OpenAIResponse,
88
+ time_elapsed: float,
89
+ ) -> trace_tree.WBTraceTree:
90
+ """Resolves the request and response objects for `openai.Edit`."""
91
+ request_str = (
92
+ f"\n\n**Instruction**: {request['instruction']}\n\n"
93
+ f"**Input**: {request['input']}\n"
94
+ )
95
+ choices = [
96
+ f"\n\n**Edited**: {choice['text']}\n" for choice in response["choices"]
97
+ ]
98
+
99
+ return self._request_response_result_to_trace(
100
+ request=request,
101
+ response=response,
102
+ request_str=request_str,
103
+ choices=choices,
104
+ time_elapsed=time_elapsed,
105
+ )
106
+
107
+ def _resolve_completion(
108
+ self,
109
+ request: Dict[str, Any],
110
+ response: OpenAIResponse,
111
+ time_elapsed: float,
112
+ ) -> trace_tree.WBTraceTree:
113
+ """Resolves the request and response objects for `openai.Completion`."""
114
+ request_str = f"\n\n**Prompt**: {request['prompt']}\n"
115
+ choices = [
116
+ f"\n\n**Completion**: {choice['text']}\n"
117
+ for choice in response["choices"]
118
+ ]
119
+
120
+ return self._request_response_result_to_trace(
121
+ request=request,
122
+ response=response,
123
+ request_str=request_str,
124
+ choices=choices,
125
+ time_elapsed=time_elapsed,
126
+ )
127
+
128
+ def _resolve_chat_completion(
129
+ self,
130
+ request: Dict[str, Any],
131
+ response: OpenAIResponse,
132
+ time_elapsed: float,
133
+ ) -> trace_tree.WBTraceTree:
134
+ """Resolves the request and response objects for `openai.Completion`."""
135
+ prompt = io.StringIO()
136
+ for message in request["messages"]:
137
+ prompt.write(f"\n\n**{message['role']}**: {message['content']}\n")
138
+ request_str = prompt.getvalue()
139
+
140
+ choices = [
141
+ f"\n\n**{choice['message']['role']}**: {choice['message']['content']}\n"
142
+ for choice in response["choices"]
143
+ ]
144
+
145
+ return self._request_response_result_to_trace(
146
+ request=request,
147
+ response=response,
148
+ request_str=request_str,
149
+ choices=choices,
150
+ time_elapsed=time_elapsed,
151
+ )
152
+
153
+ def _request_response_result_to_trace(
154
+ self,
155
+ request: Dict[str, Any],
156
+ response: OpenAIResponse,
157
+ request_str: str,
158
+ choices: List[str],
159
+ time_elapsed: float,
160
+ ) -> trace_tree.WBTraceTree:
161
+ """Resolves the request and response objects for `openai.Completion`."""
162
+ results = [
163
+ trace_tree.Result(
164
+ inputs={"request": request_str},
165
+ outputs={"response": choice},
166
+ )
167
+ for choice in choices
168
+ ]
169
+ trace = self.results_to_trace_tree(request, response, results, time_elapsed)
170
+ return trace
171
+
172
+ except Exception:
173
+ imported_openAIResponse = False
174
+
175
+
176
+ #### What this does ####
177
+ # On success, logs events to Langfuse
178
+ import traceback
179
+
180
+
181
+ class WeightsBiasesLogger:
182
+ # Class variables or attributes
183
+ def __init__(self):
184
+ try:
185
+ pass
186
+ except Exception:
187
+ raise Exception(
188
+ "\033[91m wandb not installed, try running 'pip install wandb' to fix this error\033[0m"
189
+ )
190
+ if imported_openAIResponse is False:
191
+ raise Exception(
192
+ "\033[91m wandb not installed, try running 'pip install wandb' to fix this error\033[0m"
193
+ )
194
+ self.resolver = OpenAIRequestResponseResolver()
195
+
196
+ def log_event(self, kwargs, response_obj, start_time, end_time, print_verbose):
197
+ # Method definition
198
+ import wandb
199
+
200
+ try:
201
+ print_verbose(f"W&B Logging - Enters logging function for model {kwargs}")
202
+ run = wandb.init()
203
+ print_verbose(response_obj)
204
+
205
+ trace = self.resolver(
206
+ kwargs, response_obj, (end_time - start_time).total_seconds()
207
+ )
208
+
209
+ if trace is not None and run is not None:
210
+ run.log({"trace": trace})
211
+
212
+ if run is not None:
213
+ run.finish()
214
+ print_verbose(
215
+ f"W&B Logging Logging - final response object: {response_obj}"
216
+ )
217
+ except Exception:
218
+ print_verbose(f"W&B Logging Layer Error - {traceback.format_exc()}")
219
+ pass
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/__init__.py ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ LiteLLM Interactions API
3
+
4
+ This module provides SDK methods for Google's Interactions API.
5
+
6
+ Usage:
7
+ import litellm
8
+
9
+ # Create an interaction with a model
10
+ response = litellm.interactions.create(
11
+ model="gemini-2.5-flash",
12
+ input="Hello, how are you?"
13
+ )
14
+
15
+ # Create an interaction with an agent
16
+ response = litellm.interactions.create(
17
+ agent="deep-research-pro-preview-12-2025",
18
+ input="Research the current state of cancer research"
19
+ )
20
+
21
+ # Async version
22
+ response = await litellm.interactions.acreate(...)
23
+
24
+ # Get an interaction
25
+ response = litellm.interactions.get(interaction_id="...")
26
+
27
+ # Delete an interaction
28
+ result = litellm.interactions.delete(interaction_id="...")
29
+
30
+ # Cancel an interaction
31
+ result = litellm.interactions.cancel(interaction_id="...")
32
+
33
+ Methods:
34
+ - create(): Sync create interaction
35
+ - acreate(): Async create interaction
36
+ - get(): Sync get interaction
37
+ - aget(): Async get interaction
38
+ - delete(): Sync delete interaction
39
+ - adelete(): Async delete interaction
40
+ - cancel(): Sync cancel interaction
41
+ - acancel(): Async cancel interaction
42
+ """
43
+
44
+ from litellm.interactions.main import (
45
+ acancel,
46
+ acreate,
47
+ adelete,
48
+ aget,
49
+ cancel,
50
+ create,
51
+ delete,
52
+ get,
53
+ )
54
+
55
+ __all__ = [
56
+ # Create
57
+ "create",
58
+ "acreate",
59
+ # Get
60
+ "get",
61
+ "aget",
62
+ # Delete
63
+ "delete",
64
+ "adelete",
65
+ # Cancel
66
+ "cancel",
67
+ "acancel",
68
+ ]
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/http_handler.py ADDED
@@ -0,0 +1,697 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ HTTP Handler for Interactions API requests.
3
+
4
+ This module handles the HTTP communication for the Google Interactions API.
5
+ """
6
+
7
+ from typing import (
8
+ Any,
9
+ AsyncIterator,
10
+ Coroutine,
11
+ Dict,
12
+ Iterator,
13
+ Optional,
14
+ Union,
15
+ )
16
+
17
+ import httpx
18
+
19
+ import litellm
20
+ from litellm.constants import request_timeout
21
+ from litellm.interactions.streaming_iterator import (
22
+ InteractionsAPIStreamingIterator,
23
+ SyncInteractionsAPIStreamingIterator,
24
+ )
25
+ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
26
+ from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
27
+ from litellm.llms.custom_httpx.http_handler import (
28
+ AsyncHTTPHandler,
29
+ HTTPHandler,
30
+ _get_httpx_client,
31
+ get_async_httpx_client,
32
+ )
33
+ from litellm.types.interactions import (
34
+ CancelInteractionResult,
35
+ DeleteInteractionResult,
36
+ InteractionInput,
37
+ InteractionsAPIOptionalRequestParams,
38
+ InteractionsAPIResponse,
39
+ InteractionsAPIStreamingResponse,
40
+ )
41
+ from litellm.types.router import GenericLiteLLMParams
42
+
43
+
44
+ class InteractionsHTTPHandler:
45
+ """
46
+ HTTP handler for Interactions API requests.
47
+ """
48
+
49
+ def _handle_error(
50
+ self,
51
+ e: Exception,
52
+ provider_config: BaseInteractionsAPIConfig,
53
+ ) -> Exception:
54
+ """Handle errors from HTTP requests."""
55
+ if isinstance(e, httpx.HTTPStatusError):
56
+ error_message = e.response.text
57
+ status_code = e.response.status_code
58
+ headers = dict(e.response.headers)
59
+ return provider_config.get_error_class(
60
+ error_message=error_message,
61
+ status_code=status_code,
62
+ headers=headers,
63
+ )
64
+ return e
65
+
66
+ # =========================================================
67
+ # CREATE INTERACTION
68
+ # =========================================================
69
+
70
+ def create_interaction(
71
+ self,
72
+ interactions_api_config: BaseInteractionsAPIConfig,
73
+ optional_params: InteractionsAPIOptionalRequestParams,
74
+ custom_llm_provider: str,
75
+ litellm_params: GenericLiteLLMParams,
76
+ logging_obj: LiteLLMLoggingObj,
77
+ model: Optional[str] = None,
78
+ agent: Optional[str] = None,
79
+ input: Optional[InteractionInput] = None,
80
+ extra_headers: Optional[Dict[str, Any]] = None,
81
+ extra_body: Optional[Dict[str, Any]] = None,
82
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
83
+ client: Optional[HTTPHandler] = None,
84
+ _is_async: bool = False,
85
+ stream: Optional[bool] = None,
86
+ ) -> Union[
87
+ InteractionsAPIResponse,
88
+ Iterator[InteractionsAPIStreamingResponse],
89
+ Coroutine[
90
+ Any,
91
+ Any,
92
+ Union[
93
+ InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]
94
+ ],
95
+ ],
96
+ ]:
97
+ """
98
+ Create a new interaction (synchronous or async based on _is_async flag).
99
+
100
+ Per Google's OpenAPI spec, the endpoint is POST /{api_version}/interactions
101
+ """
102
+ if _is_async:
103
+ return self.async_create_interaction(
104
+ model=model,
105
+ agent=agent,
106
+ input=input,
107
+ interactions_api_config=interactions_api_config,
108
+ optional_params=optional_params,
109
+ custom_llm_provider=custom_llm_provider,
110
+ litellm_params=litellm_params,
111
+ logging_obj=logging_obj,
112
+ extra_headers=extra_headers,
113
+ extra_body=extra_body,
114
+ timeout=timeout,
115
+ stream=stream,
116
+ )
117
+
118
+ if client is None:
119
+ sync_httpx_client = _get_httpx_client(
120
+ params={"ssl_verify": litellm_params.get("ssl_verify", None)}
121
+ )
122
+ else:
123
+ sync_httpx_client = client
124
+
125
+ headers = interactions_api_config.validate_environment(
126
+ headers=extra_headers or {},
127
+ model=model or "",
128
+ litellm_params=litellm_params,
129
+ )
130
+
131
+ api_base = interactions_api_config.get_complete_url(
132
+ api_base=litellm_params.api_base or "",
133
+ model=model,
134
+ agent=agent,
135
+ litellm_params=dict(litellm_params),
136
+ stream=stream,
137
+ )
138
+
139
+ data = interactions_api_config.transform_request(
140
+ model=model,
141
+ agent=agent,
142
+ input=input,
143
+ optional_params=optional_params,
144
+ litellm_params=litellm_params,
145
+ headers=headers,
146
+ )
147
+
148
+ if extra_body:
149
+ data.update(extra_body)
150
+
151
+ # Logging
152
+ logging_obj.pre_call(
153
+ input=input,
154
+ api_key="",
155
+ additional_args={
156
+ "complete_input_dict": data,
157
+ "api_base": api_base,
158
+ "headers": headers,
159
+ },
160
+ )
161
+
162
+ try:
163
+ if stream:
164
+ response = sync_httpx_client.post(
165
+ url=api_base,
166
+ headers=headers,
167
+ json=data,
168
+ timeout=timeout or request_timeout,
169
+ stream=True,
170
+ )
171
+ return self._create_sync_streaming_iterator(
172
+ response=response,
173
+ model=model,
174
+ logging_obj=logging_obj,
175
+ interactions_api_config=interactions_api_config,
176
+ )
177
+ else:
178
+ response = sync_httpx_client.post(
179
+ url=api_base,
180
+ headers=headers,
181
+ json=data,
182
+ timeout=timeout or request_timeout,
183
+ )
184
+ except Exception as e:
185
+ raise self._handle_error(e=e, provider_config=interactions_api_config)
186
+
187
+ return interactions_api_config.transform_response(
188
+ model=model,
189
+ raw_response=response,
190
+ logging_obj=logging_obj,
191
+ )
192
+
193
+ async def async_create_interaction(
194
+ self,
195
+ interactions_api_config: BaseInteractionsAPIConfig,
196
+ optional_params: InteractionsAPIOptionalRequestParams,
197
+ custom_llm_provider: str,
198
+ litellm_params: GenericLiteLLMParams,
199
+ logging_obj: LiteLLMLoggingObj,
200
+ model: Optional[str] = None,
201
+ agent: Optional[str] = None,
202
+ input: Optional[InteractionInput] = None,
203
+ extra_headers: Optional[Dict[str, Any]] = None,
204
+ extra_body: Optional[Dict[str, Any]] = None,
205
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
206
+ client: Optional[AsyncHTTPHandler] = None,
207
+ stream: Optional[bool] = None,
208
+ ) -> Union[
209
+ InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]
210
+ ]:
211
+ """
212
+ Create a new interaction (async version).
213
+ """
214
+ if client is None:
215
+ async_httpx_client = get_async_httpx_client(
216
+ llm_provider=litellm.LlmProviders(custom_llm_provider),
217
+ params={"ssl_verify": litellm_params.get("ssl_verify", None)},
218
+ )
219
+ else:
220
+ async_httpx_client = client
221
+
222
+ headers = interactions_api_config.validate_environment(
223
+ headers=extra_headers or {},
224
+ model=model or "",
225
+ litellm_params=litellm_params,
226
+ )
227
+
228
+ api_base = interactions_api_config.get_complete_url(
229
+ api_base=litellm_params.api_base or "",
230
+ model=model,
231
+ agent=agent,
232
+ litellm_params=dict(litellm_params),
233
+ stream=stream,
234
+ )
235
+
236
+ data = interactions_api_config.transform_request(
237
+ model=model,
238
+ agent=agent,
239
+ input=input,
240
+ optional_params=optional_params,
241
+ litellm_params=litellm_params,
242
+ headers=headers,
243
+ )
244
+
245
+ if extra_body:
246
+ data.update(extra_body)
247
+
248
+ # Logging
249
+ logging_obj.pre_call(
250
+ input=input,
251
+ api_key="",
252
+ additional_args={
253
+ "complete_input_dict": data,
254
+ "api_base": api_base,
255
+ "headers": headers,
256
+ },
257
+ )
258
+
259
+ try:
260
+ if stream:
261
+ response = await async_httpx_client.post(
262
+ url=api_base,
263
+ headers=headers,
264
+ json=data,
265
+ timeout=timeout or request_timeout,
266
+ stream=True,
267
+ )
268
+ return self._create_async_streaming_iterator(
269
+ response=response,
270
+ model=model,
271
+ logging_obj=logging_obj,
272
+ interactions_api_config=interactions_api_config,
273
+ )
274
+ else:
275
+ response = await async_httpx_client.post(
276
+ url=api_base,
277
+ headers=headers,
278
+ json=data,
279
+ timeout=timeout or request_timeout,
280
+ )
281
+ except Exception as e:
282
+ raise self._handle_error(e=e, provider_config=interactions_api_config)
283
+
284
+ return interactions_api_config.transform_response(
285
+ model=model,
286
+ raw_response=response,
287
+ logging_obj=logging_obj,
288
+ )
289
+
290
+ def _create_sync_streaming_iterator(
291
+ self,
292
+ response: httpx.Response,
293
+ model: Optional[str],
294
+ logging_obj: LiteLLMLoggingObj,
295
+ interactions_api_config: BaseInteractionsAPIConfig,
296
+ ) -> SyncInteractionsAPIStreamingIterator:
297
+ """Create a synchronous streaming iterator.
298
+
299
+ Google AI's streaming format uses SSE (Server-Sent Events).
300
+ Returns a proper streaming iterator that yields chunks as they arrive.
301
+ """
302
+ return SyncInteractionsAPIStreamingIterator(
303
+ response=response,
304
+ model=model,
305
+ interactions_api_config=interactions_api_config,
306
+ logging_obj=logging_obj,
307
+ )
308
+
309
+ def _create_async_streaming_iterator(
310
+ self,
311
+ response: httpx.Response,
312
+ model: Optional[str],
313
+ logging_obj: LiteLLMLoggingObj,
314
+ interactions_api_config: BaseInteractionsAPIConfig,
315
+ ) -> InteractionsAPIStreamingIterator:
316
+ """Create an asynchronous streaming iterator.
317
+
318
+ Google AI's streaming format uses SSE (Server-Sent Events).
319
+ Returns a proper streaming iterator that yields chunks as they arrive.
320
+ """
321
+ return InteractionsAPIStreamingIterator(
322
+ response=response,
323
+ model=model,
324
+ interactions_api_config=interactions_api_config,
325
+ logging_obj=logging_obj,
326
+ )
327
+
328
+ # =========================================================
329
+ # GET INTERACTION
330
+ # =========================================================
331
+
332
+ def get_interaction(
333
+ self,
334
+ interaction_id: str,
335
+ interactions_api_config: BaseInteractionsAPIConfig,
336
+ custom_llm_provider: str,
337
+ litellm_params: GenericLiteLLMParams,
338
+ logging_obj: LiteLLMLoggingObj,
339
+ extra_headers: Optional[Dict[str, Any]] = None,
340
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
341
+ client: Optional[HTTPHandler] = None,
342
+ _is_async: bool = False,
343
+ ) -> Union[InteractionsAPIResponse, Coroutine[Any, Any, InteractionsAPIResponse]]:
344
+ """Get an interaction by ID."""
345
+ if _is_async:
346
+ return self.async_get_interaction(
347
+ interaction_id=interaction_id,
348
+ interactions_api_config=interactions_api_config,
349
+ custom_llm_provider=custom_llm_provider,
350
+ litellm_params=litellm_params,
351
+ logging_obj=logging_obj,
352
+ extra_headers=extra_headers,
353
+ timeout=timeout,
354
+ )
355
+
356
+ if client is None:
357
+ sync_httpx_client = _get_httpx_client(
358
+ params={"ssl_verify": litellm_params.get("ssl_verify", None)}
359
+ )
360
+ else:
361
+ sync_httpx_client = client
362
+
363
+ headers = interactions_api_config.validate_environment(
364
+ headers=extra_headers or {},
365
+ model="",
366
+ litellm_params=litellm_params,
367
+ )
368
+
369
+ url, params = interactions_api_config.transform_get_interaction_request(
370
+ interaction_id=interaction_id,
371
+ api_base=litellm_params.api_base or "",
372
+ litellm_params=litellm_params,
373
+ headers=headers,
374
+ )
375
+
376
+ logging_obj.pre_call(
377
+ input=interaction_id,
378
+ api_key="",
379
+ additional_args={"api_base": url, "headers": headers},
380
+ )
381
+
382
+ try:
383
+ response = sync_httpx_client.get(
384
+ url=url,
385
+ headers=headers,
386
+ params=params,
387
+ )
388
+ except Exception as e:
389
+ raise self._handle_error(e=e, provider_config=interactions_api_config)
390
+
391
+ return interactions_api_config.transform_get_interaction_response(
392
+ raw_response=response,
393
+ logging_obj=logging_obj,
394
+ )
395
+
396
+ async def async_get_interaction(
397
+ self,
398
+ interaction_id: str,
399
+ interactions_api_config: BaseInteractionsAPIConfig,
400
+ custom_llm_provider: str,
401
+ litellm_params: GenericLiteLLMParams,
402
+ logging_obj: LiteLLMLoggingObj,
403
+ extra_headers: Optional[Dict[str, Any]] = None,
404
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
405
+ client: Optional[AsyncHTTPHandler] = None,
406
+ ) -> InteractionsAPIResponse:
407
+ """Get an interaction by ID (async version)."""
408
+ if client is None:
409
+ async_httpx_client = get_async_httpx_client(
410
+ llm_provider=litellm.LlmProviders(custom_llm_provider),
411
+ params={"ssl_verify": litellm_params.get("ssl_verify", None)},
412
+ )
413
+ else:
414
+ async_httpx_client = client
415
+
416
+ headers = interactions_api_config.validate_environment(
417
+ headers=extra_headers or {},
418
+ model="",
419
+ litellm_params=litellm_params,
420
+ )
421
+
422
+ url, params = interactions_api_config.transform_get_interaction_request(
423
+ interaction_id=interaction_id,
424
+ api_base=litellm_params.api_base or "",
425
+ litellm_params=litellm_params,
426
+ headers=headers,
427
+ )
428
+
429
+ logging_obj.pre_call(
430
+ input=interaction_id,
431
+ api_key="",
432
+ additional_args={"api_base": url, "headers": headers},
433
+ )
434
+
435
+ try:
436
+ response = await async_httpx_client.get(
437
+ url=url,
438
+ headers=headers,
439
+ params=params,
440
+ )
441
+ except Exception as e:
442
+ raise self._handle_error(e=e, provider_config=interactions_api_config)
443
+
444
+ return interactions_api_config.transform_get_interaction_response(
445
+ raw_response=response,
446
+ logging_obj=logging_obj,
447
+ )
448
+
449
+ # =========================================================
450
+ # DELETE INTERACTION
451
+ # =========================================================
452
+
453
+ def delete_interaction(
454
+ self,
455
+ interaction_id: str,
456
+ interactions_api_config: BaseInteractionsAPIConfig,
457
+ custom_llm_provider: str,
458
+ litellm_params: GenericLiteLLMParams,
459
+ logging_obj: LiteLLMLoggingObj,
460
+ extra_headers: Optional[Dict[str, Any]] = None,
461
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
462
+ client: Optional[HTTPHandler] = None,
463
+ _is_async: bool = False,
464
+ ) -> Union[DeleteInteractionResult, Coroutine[Any, Any, DeleteInteractionResult]]:
465
+ """Delete an interaction by ID."""
466
+ if _is_async:
467
+ return self.async_delete_interaction(
468
+ interaction_id=interaction_id,
469
+ interactions_api_config=interactions_api_config,
470
+ custom_llm_provider=custom_llm_provider,
471
+ litellm_params=litellm_params,
472
+ logging_obj=logging_obj,
473
+ extra_headers=extra_headers,
474
+ timeout=timeout,
475
+ )
476
+
477
+ if client is None:
478
+ sync_httpx_client = _get_httpx_client(
479
+ params={"ssl_verify": litellm_params.get("ssl_verify", None)}
480
+ )
481
+ else:
482
+ sync_httpx_client = client
483
+
484
+ headers = interactions_api_config.validate_environment(
485
+ headers=extra_headers or {},
486
+ model="",
487
+ litellm_params=litellm_params,
488
+ )
489
+
490
+ url, data = interactions_api_config.transform_delete_interaction_request(
491
+ interaction_id=interaction_id,
492
+ api_base=litellm_params.api_base or "",
493
+ litellm_params=litellm_params,
494
+ headers=headers,
495
+ )
496
+
497
+ logging_obj.pre_call(
498
+ input=interaction_id,
499
+ api_key="",
500
+ additional_args={"api_base": url, "headers": headers},
501
+ )
502
+
503
+ try:
504
+ response = sync_httpx_client.delete(
505
+ url=url,
506
+ headers=headers,
507
+ timeout=timeout or request_timeout,
508
+ )
509
+ except Exception as e:
510
+ raise self._handle_error(e=e, provider_config=interactions_api_config)
511
+
512
+ return interactions_api_config.transform_delete_interaction_response(
513
+ raw_response=response,
514
+ logging_obj=logging_obj,
515
+ interaction_id=interaction_id,
516
+ )
517
+
518
+ async def async_delete_interaction(
519
+ self,
520
+ interaction_id: str,
521
+ interactions_api_config: BaseInteractionsAPIConfig,
522
+ custom_llm_provider: str,
523
+ litellm_params: GenericLiteLLMParams,
524
+ logging_obj: LiteLLMLoggingObj,
525
+ extra_headers: Optional[Dict[str, Any]] = None,
526
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
527
+ client: Optional[AsyncHTTPHandler] = None,
528
+ ) -> DeleteInteractionResult:
529
+ """Delete an interaction by ID (async version)."""
530
+ if client is None:
531
+ async_httpx_client = get_async_httpx_client(
532
+ llm_provider=litellm.LlmProviders(custom_llm_provider),
533
+ params={"ssl_verify": litellm_params.get("ssl_verify", None)},
534
+ )
535
+ else:
536
+ async_httpx_client = client
537
+
538
+ headers = interactions_api_config.validate_environment(
539
+ headers=extra_headers or {},
540
+ model="",
541
+ litellm_params=litellm_params,
542
+ )
543
+
544
+ url, data = interactions_api_config.transform_delete_interaction_request(
545
+ interaction_id=interaction_id,
546
+ api_base=litellm_params.api_base or "",
547
+ litellm_params=litellm_params,
548
+ headers=headers,
549
+ )
550
+
551
+ logging_obj.pre_call(
552
+ input=interaction_id,
553
+ api_key="",
554
+ additional_args={"api_base": url, "headers": headers},
555
+ )
556
+
557
+ try:
558
+ response = await async_httpx_client.delete(
559
+ url=url,
560
+ headers=headers,
561
+ timeout=timeout or request_timeout,
562
+ )
563
+ except Exception as e:
564
+ raise self._handle_error(e=e, provider_config=interactions_api_config)
565
+
566
+ return interactions_api_config.transform_delete_interaction_response(
567
+ raw_response=response,
568
+ logging_obj=logging_obj,
569
+ interaction_id=interaction_id,
570
+ )
571
+
572
+ # =========================================================
573
+ # CANCEL INTERACTION
574
+ # =========================================================
575
+
576
+ def cancel_interaction(
577
+ self,
578
+ interaction_id: str,
579
+ interactions_api_config: BaseInteractionsAPIConfig,
580
+ custom_llm_provider: str,
581
+ litellm_params: GenericLiteLLMParams,
582
+ logging_obj: LiteLLMLoggingObj,
583
+ extra_headers: Optional[Dict[str, Any]] = None,
584
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
585
+ client: Optional[HTTPHandler] = None,
586
+ _is_async: bool = False,
587
+ ) -> Union[CancelInteractionResult, Coroutine[Any, Any, CancelInteractionResult]]:
588
+ """Cancel an interaction by ID."""
589
+ if _is_async:
590
+ return self.async_cancel_interaction(
591
+ interaction_id=interaction_id,
592
+ interactions_api_config=interactions_api_config,
593
+ custom_llm_provider=custom_llm_provider,
594
+ litellm_params=litellm_params,
595
+ logging_obj=logging_obj,
596
+ extra_headers=extra_headers,
597
+ timeout=timeout,
598
+ )
599
+
600
+ if client is None:
601
+ sync_httpx_client = _get_httpx_client(
602
+ params={"ssl_verify": litellm_params.get("ssl_verify", None)}
603
+ )
604
+ else:
605
+ sync_httpx_client = client
606
+
607
+ headers = interactions_api_config.validate_environment(
608
+ headers=extra_headers or {},
609
+ model="",
610
+ litellm_params=litellm_params,
611
+ )
612
+
613
+ url, data = interactions_api_config.transform_cancel_interaction_request(
614
+ interaction_id=interaction_id,
615
+ api_base=litellm_params.api_base or "",
616
+ litellm_params=litellm_params,
617
+ headers=headers,
618
+ )
619
+
620
+ logging_obj.pre_call(
621
+ input=interaction_id,
622
+ api_key="",
623
+ additional_args={"api_base": url, "headers": headers},
624
+ )
625
+
626
+ try:
627
+ response = sync_httpx_client.post(
628
+ url=url,
629
+ headers=headers,
630
+ json=data,
631
+ timeout=timeout or request_timeout,
632
+ )
633
+ except Exception as e:
634
+ raise self._handle_error(e=e, provider_config=interactions_api_config)
635
+
636
+ return interactions_api_config.transform_cancel_interaction_response(
637
+ raw_response=response,
638
+ logging_obj=logging_obj,
639
+ )
640
+
641
+ async def async_cancel_interaction(
642
+ self,
643
+ interaction_id: str,
644
+ interactions_api_config: BaseInteractionsAPIConfig,
645
+ custom_llm_provider: str,
646
+ litellm_params: GenericLiteLLMParams,
647
+ logging_obj: LiteLLMLoggingObj,
648
+ extra_headers: Optional[Dict[str, Any]] = None,
649
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
650
+ client: Optional[AsyncHTTPHandler] = None,
651
+ ) -> CancelInteractionResult:
652
+ """Cancel an interaction by ID (async version)."""
653
+ if client is None:
654
+ async_httpx_client = get_async_httpx_client(
655
+ llm_provider=litellm.LlmProviders(custom_llm_provider),
656
+ params={"ssl_verify": litellm_params.get("ssl_verify", None)},
657
+ )
658
+ else:
659
+ async_httpx_client = client
660
+
661
+ headers = interactions_api_config.validate_environment(
662
+ headers=extra_headers or {},
663
+ model="",
664
+ litellm_params=litellm_params,
665
+ )
666
+
667
+ url, data = interactions_api_config.transform_cancel_interaction_request(
668
+ interaction_id=interaction_id,
669
+ api_base=litellm_params.api_base or "",
670
+ litellm_params=litellm_params,
671
+ headers=headers,
672
+ )
673
+
674
+ logging_obj.pre_call(
675
+ input=interaction_id,
676
+ api_key="",
677
+ additional_args={"api_base": url, "headers": headers},
678
+ )
679
+
680
+ try:
681
+ response = await async_httpx_client.post(
682
+ url=url,
683
+ headers=headers,
684
+ json=data,
685
+ timeout=timeout or request_timeout,
686
+ )
687
+ except Exception as e:
688
+ raise self._handle_error(e=e, provider_config=interactions_api_config)
689
+
690
+ return interactions_api_config.transform_cancel_interaction_response(
691
+ raw_response=response,
692
+ logging_obj=logging_obj,
693
+ )
694
+
695
+
696
+ # Initialize the HTTP handler singleton
697
+ interactions_http_handler = InteractionsHTTPHandler()
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/__init__.py ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Bridge module for connecting Interactions API to Responses API via litellm.responses().
3
+ """
4
+
5
+ from litellm.interactions.litellm_responses_transformation.handler import (
6
+ LiteLLMResponsesInteractionsHandler,
7
+ )
8
+ from litellm.interactions.litellm_responses_transformation.transformation import (
9
+ LiteLLMResponsesInteractionsConfig,
10
+ )
11
+
12
+ __all__ = [
13
+ "LiteLLMResponsesInteractionsHandler",
14
+ "LiteLLMResponsesInteractionsConfig", # Transformation config class (not BaseInteractionsAPIConfig)
15
+ ]
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/handler.py ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Handler for transforming interactions API requests to litellm.responses requests.
3
+ """
4
+
5
+ from typing import (
6
+ Any,
7
+ AsyncIterator,
8
+ Coroutine,
9
+ Dict,
10
+ Iterator,
11
+ Optional,
12
+ Union,
13
+ cast,
14
+ )
15
+
16
+ import litellm
17
+ from litellm.interactions.litellm_responses_transformation.streaming_iterator import (
18
+ LiteLLMResponsesInteractionsStreamingIterator,
19
+ )
20
+ from litellm.interactions.litellm_responses_transformation.transformation import (
21
+ LiteLLMResponsesInteractionsConfig,
22
+ )
23
+ from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
24
+ from litellm.types.interactions import (
25
+ InteractionInput,
26
+ InteractionsAPIOptionalRequestParams,
27
+ InteractionsAPIResponse,
28
+ InteractionsAPIStreamingResponse,
29
+ )
30
+ from litellm.types.llms.openai import ResponsesAPIResponse
31
+
32
+
33
+ class LiteLLMResponsesInteractionsHandler:
34
+ """Handler for bridging Interactions API to Responses API via litellm.responses()."""
35
+
36
+ def interactions_api_handler(
37
+ self,
38
+ model: str,
39
+ input: Optional[InteractionInput],
40
+ optional_params: InteractionsAPIOptionalRequestParams,
41
+ custom_llm_provider: Optional[str] = None,
42
+ _is_async: bool = False,
43
+ stream: Optional[bool] = None,
44
+ **kwargs,
45
+ ) -> Union[
46
+ InteractionsAPIResponse,
47
+ Iterator[InteractionsAPIStreamingResponse],
48
+ Coroutine[
49
+ Any,
50
+ Any,
51
+ Union[
52
+ InteractionsAPIResponse,
53
+ AsyncIterator[InteractionsAPIStreamingResponse],
54
+ ],
55
+ ],
56
+ ]:
57
+ """
58
+ Handle Interactions API request by calling litellm.responses().
59
+
60
+ Args:
61
+ model: The model to use
62
+ input: The input content
63
+ optional_params: Optional parameters for the request
64
+ custom_llm_provider: Override LLM provider
65
+ _is_async: Whether this is an async call
66
+ stream: Whether to stream the response
67
+ **kwargs: Additional parameters
68
+
69
+ Returns:
70
+ InteractionsAPIResponse or streaming iterator
71
+ """
72
+ # Transform interactions request to responses request
73
+ responses_request = LiteLLMResponsesInteractionsConfig.transform_interactions_request_to_responses_request(
74
+ model=model,
75
+ input=input,
76
+ optional_params=optional_params,
77
+ custom_llm_provider=custom_llm_provider,
78
+ stream=stream,
79
+ **kwargs,
80
+ )
81
+
82
+ if _is_async:
83
+ return self.async_interactions_api_handler(
84
+ responses_request=responses_request,
85
+ model=model,
86
+ input=input,
87
+ optional_params=optional_params,
88
+ **kwargs,
89
+ )
90
+
91
+ # Call litellm.responses()
92
+ # Note: litellm.responses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]
93
+ # but the type checker may see it as a coroutine in some contexts
94
+ responses_response = litellm.responses(
95
+ **responses_request,
96
+ )
97
+
98
+ # Handle streaming response
99
+ if isinstance(responses_response, BaseResponsesAPIStreamingIterator):
100
+ return LiteLLMResponsesInteractionsStreamingIterator(
101
+ model=model,
102
+ litellm_custom_stream_wrapper=responses_response,
103
+ request_input=input,
104
+ optional_params=optional_params,
105
+ custom_llm_provider=custom_llm_provider,
106
+ litellm_metadata=kwargs.get("litellm_metadata", {}),
107
+ )
108
+
109
+ # At this point, responses_response must be ResponsesAPIResponse (not streaming)
110
+ # Cast to satisfy type checker since we've already checked it's not a streaming iterator
111
+ responses_api_response = cast(ResponsesAPIResponse, responses_response)
112
+
113
+ # Transform responses response to interactions response
114
+ return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
115
+ responses_response=responses_api_response,
116
+ model=model,
117
+ )
118
+
119
+ async def async_interactions_api_handler(
120
+ self,
121
+ responses_request: Dict[str, Any],
122
+ model: str,
123
+ input: Optional[InteractionInput],
124
+ optional_params: InteractionsAPIOptionalRequestParams,
125
+ **kwargs,
126
+ ) -> Union[
127
+ InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]
128
+ ]:
129
+ """Async handler for interactions API requests."""
130
+ # Call litellm.aresponses()
131
+ # Note: litellm.aresponses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator]
132
+ responses_response = await litellm.aresponses(
133
+ **responses_request,
134
+ )
135
+
136
+ # Handle streaming response
137
+ if isinstance(responses_response, BaseResponsesAPIStreamingIterator):
138
+ return LiteLLMResponsesInteractionsStreamingIterator(
139
+ model=model,
140
+ litellm_custom_stream_wrapper=responses_response,
141
+ request_input=input,
142
+ optional_params=optional_params,
143
+ custom_llm_provider=responses_request.get("custom_llm_provider"),
144
+ litellm_metadata=kwargs.get("litellm_metadata", {}),
145
+ )
146
+
147
+ # At this point, responses_response must be ResponsesAPIResponse (not streaming)
148
+ # Cast to satisfy type checker since we've already checked it's not a streaming iterator
149
+ responses_api_response = cast(ResponsesAPIResponse, responses_response)
150
+
151
+ # Transform responses response to interactions response
152
+ return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response(
153
+ responses_response=responses_api_response,
154
+ model=model,
155
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/streaming_iterator.py ADDED
@@ -0,0 +1,286 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Streaming iterator for transforming Responses API stream to Interactions API stream.
3
+ """
4
+
5
+ from typing import Any, AsyncIterator, Dict, Iterator, Optional, cast
6
+
7
+ from litellm.responses.streaming_iterator import (
8
+ BaseResponsesAPIStreamingIterator,
9
+ ResponsesAPIStreamingIterator,
10
+ SyncResponsesAPIStreamingIterator,
11
+ )
12
+ from litellm.types.interactions import (
13
+ InteractionInput,
14
+ InteractionsAPIOptionalRequestParams,
15
+ InteractionsAPIStreamingResponse,
16
+ )
17
+ from litellm.types.llms.openai import (
18
+ OutputTextDeltaEvent,
19
+ ResponseCompletedEvent,
20
+ ResponseCreatedEvent,
21
+ ResponseInProgressEvent,
22
+ ResponsesAPIStreamingResponse,
23
+ )
24
+
25
+
26
+ class LiteLLMResponsesInteractionsStreamingIterator:
27
+ """
28
+ Iterator that wraps Responses API streaming and transforms chunks to Interactions API format.
29
+
30
+ This class handles both sync and async iteration, transforming Responses API
31
+ streaming events (output.text.delta, response.completed, etc.) to Interactions
32
+ API streaming events (content.delta, interaction.complete, etc.).
33
+ """
34
+
35
+ def __init__(
36
+ self,
37
+ model: str,
38
+ litellm_custom_stream_wrapper: BaseResponsesAPIStreamingIterator,
39
+ request_input: Optional[InteractionInput],
40
+ optional_params: InteractionsAPIOptionalRequestParams,
41
+ custom_llm_provider: Optional[str] = None,
42
+ litellm_metadata: Optional[Dict[str, Any]] = None,
43
+ ):
44
+ self.model = model
45
+ self.responses_stream_iterator = litellm_custom_stream_wrapper
46
+ self.request_input = request_input
47
+ self.optional_params = optional_params
48
+ self.custom_llm_provider = custom_llm_provider
49
+ self.litellm_metadata = litellm_metadata or {}
50
+ self.finished = False
51
+ self.collected_text = ""
52
+ self.sent_interaction_start = False
53
+ self.sent_content_start = False
54
+
55
+ def _transform_responses_chunk_to_interactions_chunk(
56
+ self,
57
+ responses_chunk: ResponsesAPIStreamingResponse,
58
+ ) -> Optional[InteractionsAPIStreamingResponse]:
59
+ """
60
+ Transform a Responses API streaming chunk to an Interactions API streaming chunk.
61
+
62
+ Responses API events:
63
+ - output.text.delta -> content.delta
64
+ - response.completed -> interaction.complete
65
+
66
+ Interactions API events:
67
+ - interaction.start
68
+ - content.start
69
+ - content.delta
70
+ - content.stop
71
+ - interaction.complete
72
+ """
73
+ if not responses_chunk:
74
+ return None
75
+
76
+ # Handle OutputTextDeltaEvent -> content.delta
77
+ if isinstance(responses_chunk, OutputTextDeltaEvent):
78
+ delta_text = (
79
+ responses_chunk.delta if isinstance(responses_chunk.delta, str) else ""
80
+ )
81
+ self.collected_text += delta_text
82
+
83
+ # Send interaction.start if not sent
84
+ if not self.sent_interaction_start:
85
+ self.sent_interaction_start = True
86
+ return InteractionsAPIStreamingResponse(
87
+ event_type="interaction.start",
88
+ id=getattr(responses_chunk, "item_id", None)
89
+ or f"interaction_{id(self)}",
90
+ object="interaction",
91
+ status="in_progress",
92
+ model=self.model,
93
+ )
94
+
95
+ # Send content.start if not sent
96
+ if not self.sent_content_start:
97
+ self.sent_content_start = True
98
+ return InteractionsAPIStreamingResponse(
99
+ event_type="content.start",
100
+ id=getattr(responses_chunk, "item_id", None),
101
+ object="content",
102
+ delta={"type": "text", "text": ""},
103
+ )
104
+
105
+ # Send content.delta
106
+ return InteractionsAPIStreamingResponse(
107
+ event_type="content.delta",
108
+ id=getattr(responses_chunk, "item_id", None),
109
+ object="content",
110
+ delta={"text": delta_text},
111
+ )
112
+
113
+ # Handle ResponseCreatedEvent or ResponseInProgressEvent -> interaction.start
114
+ if isinstance(responses_chunk, (ResponseCreatedEvent, ResponseInProgressEvent)):
115
+ if not self.sent_interaction_start:
116
+ self.sent_interaction_start = True
117
+ response_id = (
118
+ getattr(responses_chunk.response, "id", None)
119
+ if hasattr(responses_chunk, "response")
120
+ else None
121
+ )
122
+ return InteractionsAPIStreamingResponse(
123
+ event_type="interaction.start",
124
+ id=response_id or f"interaction_{id(self)}",
125
+ object="interaction",
126
+ status="in_progress",
127
+ model=self.model,
128
+ )
129
+
130
+ # Handle ResponseCompletedEvent -> interaction.complete
131
+ if isinstance(responses_chunk, ResponseCompletedEvent):
132
+ self.finished = True
133
+ response = responses_chunk.response
134
+
135
+ # Send content.stop first if content was started
136
+ if self.sent_content_start:
137
+ # Note: We'll send this in the iterator, not here
138
+ pass
139
+
140
+ # Send interaction.complete
141
+ return InteractionsAPIStreamingResponse(
142
+ event_type="interaction.complete",
143
+ id=getattr(response, "id", None) or f"interaction_{id(self)}",
144
+ object="interaction",
145
+ status="completed",
146
+ model=self.model,
147
+ outputs=[
148
+ {
149
+ "type": "text",
150
+ "text": self.collected_text,
151
+ }
152
+ ],
153
+ )
154
+
155
+ # For other event types, return None (skip)
156
+ return None
157
+
158
+ def __iter__(self) -> Iterator[InteractionsAPIStreamingResponse]:
159
+ """Sync iterator implementation."""
160
+ return self
161
+
162
+ def __next__(self) -> InteractionsAPIStreamingResponse:
163
+ """Get next chunk in sync mode."""
164
+ if self.finished:
165
+ raise StopIteration
166
+
167
+ # Check if we have a pending interaction.complete to send
168
+ if hasattr(self, "_pending_interaction_complete"):
169
+ pending: InteractionsAPIStreamingResponse = getattr(
170
+ self, "_pending_interaction_complete"
171
+ )
172
+ delattr(self, "_pending_interaction_complete")
173
+ return pending
174
+
175
+ # Use a loop instead of recursion to avoid stack overflow
176
+ sync_iterator = cast(
177
+ SyncResponsesAPIStreamingIterator, self.responses_stream_iterator
178
+ )
179
+ while True:
180
+ try:
181
+ # Get next chunk from responses API stream
182
+ chunk = next(sync_iterator)
183
+
184
+ # Transform chunk (chunk is already a ResponsesAPIStreamingResponse)
185
+ transformed = self._transform_responses_chunk_to_interactions_chunk(
186
+ chunk
187
+ )
188
+
189
+ if transformed:
190
+ # If we finished and content was started, send content.stop before interaction.complete
191
+ if (
192
+ self.finished
193
+ and self.sent_content_start
194
+ and transformed.event_type == "interaction.complete"
195
+ ):
196
+ # Send content.stop first
197
+ content_stop = InteractionsAPIStreamingResponse(
198
+ event_type="content.stop",
199
+ id=transformed.id,
200
+ object="content",
201
+ delta={"type": "text", "text": self.collected_text},
202
+ )
203
+ # Store the interaction.complete to send next
204
+ self._pending_interaction_complete = transformed
205
+ return content_stop
206
+ return transformed
207
+
208
+ # If no transformation, continue to next chunk (loop continues)
209
+
210
+ except StopIteration:
211
+ self.finished = True
212
+
213
+ # Send final events if needed
214
+ if self.sent_content_start:
215
+ return InteractionsAPIStreamingResponse(
216
+ event_type="content.stop",
217
+ object="content",
218
+ delta={"type": "text", "text": self.collected_text},
219
+ )
220
+
221
+ raise StopIteration
222
+
223
+ def __aiter__(self) -> AsyncIterator[InteractionsAPIStreamingResponse]:
224
+ """Async iterator implementation."""
225
+ return self
226
+
227
+ async def __anext__(self) -> InteractionsAPIStreamingResponse:
228
+ """Get next chunk in async mode."""
229
+ if self.finished:
230
+ raise StopAsyncIteration
231
+
232
+ # Check if we have a pending interaction.complete to send
233
+ if hasattr(self, "_pending_interaction_complete"):
234
+ pending: InteractionsAPIStreamingResponse = getattr(
235
+ self, "_pending_interaction_complete"
236
+ )
237
+ delattr(self, "_pending_interaction_complete")
238
+ return pending
239
+
240
+ # Use a loop instead of recursion to avoid stack overflow
241
+ async_iterator = cast(
242
+ ResponsesAPIStreamingIterator, self.responses_stream_iterator
243
+ )
244
+ while True:
245
+ try:
246
+ # Get next chunk from responses API stream
247
+ chunk = await async_iterator.__anext__()
248
+
249
+ # Transform chunk (chunk is already a ResponsesAPIStreamingResponse)
250
+ transformed = self._transform_responses_chunk_to_interactions_chunk(
251
+ chunk
252
+ )
253
+
254
+ if transformed:
255
+ # If we finished and content was started, send content.stop before interaction.complete
256
+ if (
257
+ self.finished
258
+ and self.sent_content_start
259
+ and transformed.event_type == "interaction.complete"
260
+ ):
261
+ # Send content.stop first
262
+ content_stop = InteractionsAPIStreamingResponse(
263
+ event_type="content.stop",
264
+ id=transformed.id,
265
+ object="content",
266
+ delta={"type": "text", "text": self.collected_text},
267
+ )
268
+ # Store the interaction.complete to send next
269
+ self._pending_interaction_complete = transformed
270
+ return content_stop
271
+ return transformed
272
+
273
+ # If no transformation, continue to next chunk (loop continues)
274
+
275
+ except StopAsyncIteration:
276
+ self.finished = True
277
+
278
+ # Send final events if needed
279
+ if self.sent_content_start:
280
+ return InteractionsAPIStreamingResponse(
281
+ event_type="content.stop",
282
+ object="content",
283
+ delta={"type": "text", "text": self.collected_text},
284
+ )
285
+
286
+ raise StopAsyncIteration
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/litellm_responses_transformation/transformation.py ADDED
@@ -0,0 +1,299 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Transformation utilities for bridging Interactions API to Responses API.
3
+
4
+ This module handles transforming between:
5
+ - Interactions API format (Google's format with Turn[], system_instruction, etc.)
6
+ - Responses API format (OpenAI's format with input[], instructions, etc.)
7
+ """
8
+
9
+ from typing import Any, Dict, List, Optional, cast
10
+
11
+ from litellm.types.interactions import (
12
+ InteractionInput,
13
+ InteractionsAPIOptionalRequestParams,
14
+ InteractionsAPIResponse,
15
+ Turn,
16
+ )
17
+ from litellm.types.llms.openai import (
18
+ ResponseInputParam,
19
+ ResponsesAPIResponse,
20
+ )
21
+
22
+
23
+ class LiteLLMResponsesInteractionsConfig:
24
+ """Configuration class for transforming between Interactions API and Responses API."""
25
+
26
+ @staticmethod
27
+ def transform_interactions_request_to_responses_request(
28
+ model: str,
29
+ input: Optional[InteractionInput],
30
+ optional_params: InteractionsAPIOptionalRequestParams,
31
+ **kwargs,
32
+ ) -> Dict[str, Any]:
33
+ """
34
+ Transform an Interactions API request to a Responses API request.
35
+
36
+ Key transformations:
37
+ - system_instruction -> instructions
38
+ - input (string | Turn[]) -> input (ResponseInputParam)
39
+ - tools -> tools (similar format)
40
+ - generation_config -> temperature, top_p, etc.
41
+ """
42
+ responses_request: Dict[str, Any] = {
43
+ "model": model,
44
+ }
45
+
46
+ # Transform input
47
+ if input is not None:
48
+ responses_request[
49
+ "input"
50
+ ] = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
51
+ input
52
+ )
53
+
54
+ # Transform system_instruction -> instructions
55
+ if optional_params.get("system_instruction"):
56
+ responses_request["instructions"] = optional_params["system_instruction"]
57
+
58
+ # Transform tools (similar format, pass through for now)
59
+ if optional_params.get("tools"):
60
+ responses_request["tools"] = optional_params["tools"]
61
+
62
+ # Transform generation_config to temperature, top_p, etc.
63
+ generation_config = optional_params.get("generation_config")
64
+ if generation_config:
65
+ if isinstance(generation_config, dict):
66
+ if "temperature" in generation_config:
67
+ responses_request["temperature"] = generation_config["temperature"]
68
+ if "top_p" in generation_config:
69
+ responses_request["top_p"] = generation_config["top_p"]
70
+ if "top_k" in generation_config:
71
+ # Responses API doesn't have top_k, skip it
72
+ pass
73
+ if "max_output_tokens" in generation_config:
74
+ responses_request["max_output_tokens"] = generation_config[
75
+ "max_output_tokens"
76
+ ]
77
+
78
+ # Pass through other optional params that match
79
+ passthrough_params = ["stream", "store", "metadata", "user"]
80
+ for param in passthrough_params:
81
+ if param in optional_params and optional_params[param] is not None:
82
+ responses_request[param] = optional_params[param]
83
+
84
+ # Add any extra kwargs
85
+ responses_request.update(kwargs)
86
+
87
+ return responses_request
88
+
89
+ @staticmethod
90
+ def _transform_interactions_input_to_responses_input(
91
+ input: InteractionInput,
92
+ ) -> ResponseInputParam:
93
+ """
94
+ Transform Interactions API input to Responses API input format.
95
+
96
+ Interactions API input can be:
97
+ - string: "Hello"
98
+ - Turn[]: [{"role": "user", "content": [...]}]
99
+ - Content object
100
+
101
+ Responses API input is:
102
+ - string: "Hello"
103
+ - Message[]: [{"role": "user", "content": [...]}]
104
+ """
105
+ if isinstance(input, str):
106
+ # ResponseInputParam accepts str
107
+ return cast(ResponseInputParam, input)
108
+
109
+ if isinstance(input, list):
110
+ # Turn[] format - convert to Responses API Message[] format
111
+ messages = []
112
+ for turn in input:
113
+ if isinstance(turn, dict):
114
+ role = turn.get("role", "user")
115
+ content = turn.get("content", [])
116
+
117
+ # Transform content array
118
+ transformed_content = (
119
+ LiteLLMResponsesInteractionsConfig._transform_content_array(
120
+ content
121
+ )
122
+ )
123
+
124
+ messages.append(
125
+ {
126
+ "role": role,
127
+ "content": transformed_content,
128
+ }
129
+ )
130
+ elif isinstance(turn, Turn):
131
+ # Pydantic model
132
+ role = turn.role if hasattr(turn, "role") else "user"
133
+ content = turn.content if hasattr(turn, "content") else []
134
+
135
+ # Ensure content is a list for _transform_content_array
136
+ # Cast to List[Any] to handle various content types
137
+ if isinstance(content, list):
138
+ content_list: List[Any] = list(content)
139
+ elif content is not None:
140
+ content_list = [content]
141
+ else:
142
+ content_list = []
143
+
144
+ transformed_content = (
145
+ LiteLLMResponsesInteractionsConfig._transform_content_array(
146
+ content_list
147
+ )
148
+ )
149
+
150
+ messages.append(
151
+ {
152
+ "role": role,
153
+ "content": transformed_content,
154
+ }
155
+ )
156
+
157
+ return cast(ResponseInputParam, messages)
158
+
159
+ # Single content object - wrap in message
160
+ if isinstance(input, dict):
161
+ return cast(
162
+ ResponseInputParam,
163
+ [
164
+ {
165
+ "role": "user",
166
+ "content": LiteLLMResponsesInteractionsConfig._transform_content_array(
167
+ input.get("content", [])
168
+ if isinstance(input.get("content"), list)
169
+ else [input]
170
+ ),
171
+ }
172
+ ],
173
+ )
174
+
175
+ # Fallback: convert to string
176
+ return cast(ResponseInputParam, str(input))
177
+
178
+ @staticmethod
179
+ def _transform_content_array(content: List[Any]) -> List[Dict[str, Any]]:
180
+ """Transform Interactions API content array to Responses API format."""
181
+ if not isinstance(content, list):
182
+ # Single content item - wrap in array
183
+ content = [content]
184
+
185
+ transformed: List[Dict[str, Any]] = []
186
+ for item in content:
187
+ if isinstance(item, dict):
188
+ # Already in dict format, pass through
189
+ transformed.append(item)
190
+ elif isinstance(item, str):
191
+ # Plain string - wrap in text format
192
+ transformed.append({"type": "text", "text": item})
193
+ else:
194
+ # Pydantic model or other - convert to dict
195
+ if hasattr(item, "model_dump"):
196
+ dumped = item.model_dump()
197
+ if isinstance(dumped, dict):
198
+ transformed.append(dumped)
199
+ else:
200
+ # Fallback: wrap in text format
201
+ transformed.append({"type": "text", "text": str(dumped)})
202
+ elif hasattr(item, "dict"):
203
+ dumped = item.dict()
204
+ if isinstance(dumped, dict):
205
+ transformed.append(dumped)
206
+ else:
207
+ # Fallback: wrap in text format
208
+ transformed.append({"type": "text", "text": str(dumped)})
209
+ else:
210
+ # Fallback: wrap in text format
211
+ transformed.append({"type": "text", "text": str(item)})
212
+
213
+ return transformed
214
+
215
+ @staticmethod
216
+ def transform_responses_response_to_interactions_response(
217
+ responses_response: ResponsesAPIResponse,
218
+ model: Optional[str] = None,
219
+ ) -> InteractionsAPIResponse:
220
+ """
221
+ Transform a Responses API response to an Interactions API response.
222
+
223
+ Key transformations:
224
+ - Extract text from output[].content[].text
225
+ - Convert created_at (int) to created (ISO string)
226
+ - Map status
227
+ - Extract usage
228
+ """
229
+ # Extract text from outputs
230
+ outputs = []
231
+ if hasattr(responses_response, "output") and responses_response.output:
232
+ for output_item in responses_response.output:
233
+ # Use getattr with None default to safely access content
234
+ content = getattr(output_item, "content", None)
235
+ if content is not None:
236
+ content_items = content if isinstance(content, list) else [content]
237
+ for content_item in content_items:
238
+ # Check if content_item has text attribute
239
+ text = getattr(content_item, "text", None)
240
+ if text is not None:
241
+ outputs.append(
242
+ {
243
+ "type": "text",
244
+ "text": text,
245
+ }
246
+ )
247
+ elif (
248
+ isinstance(content_item, dict)
249
+ and content_item.get("type") == "text"
250
+ ):
251
+ outputs.append(content_item)
252
+
253
+ # Convert created_at to ISO string
254
+ created_at = getattr(responses_response, "created_at", None)
255
+ if isinstance(created_at, int):
256
+ from datetime import datetime
257
+
258
+ created = datetime.fromtimestamp(created_at).isoformat()
259
+ elif created_at is not None and hasattr(created_at, "isoformat"):
260
+ created = created_at.isoformat()
261
+ else:
262
+ created = None
263
+
264
+ # Map status
265
+ status = getattr(responses_response, "status", "completed")
266
+ if status == "completed":
267
+ interactions_status = "completed"
268
+ elif status == "in_progress":
269
+ interactions_status = "in_progress"
270
+ else:
271
+ interactions_status = status
272
+
273
+ # Build interactions response
274
+ interactions_response_dict: Dict[str, Any] = {
275
+ "id": getattr(responses_response, "id", ""),
276
+ "object": "interaction",
277
+ "status": interactions_status,
278
+ "outputs": outputs,
279
+ "model": model or getattr(responses_response, "model", ""),
280
+ "created": created,
281
+ }
282
+
283
+ # Add usage if available
284
+ # Map Responses API usage (input_tokens, output_tokens) to Interactions API spec format
285
+ # (total_input_tokens, total_output_tokens)
286
+ usage = getattr(responses_response, "usage", None)
287
+ if usage:
288
+ interactions_response_dict["usage"] = {
289
+ "total_input_tokens": getattr(usage, "input_tokens", 0),
290
+ "total_output_tokens": getattr(usage, "output_tokens", 0),
291
+ }
292
+
293
+ # Add role
294
+ interactions_response_dict["role"] = "model"
295
+
296
+ # Add updated (same as created for now)
297
+ interactions_response_dict["updated"] = created
298
+
299
+ return InteractionsAPIResponse(**interactions_response_dict)
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/main.py ADDED
@@ -0,0 +1,644 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ LiteLLM Interactions API - Main Module
3
+
4
+ Per OpenAPI spec (https://ai.google.dev/static/api/interactions.openapi.json):
5
+ - Create interaction: POST /{api_version}/interactions
6
+ - Get interaction: GET /{api_version}/interactions/{interaction_id}
7
+ - Delete interaction: DELETE /{api_version}/interactions/{interaction_id}
8
+
9
+ Usage:
10
+ import litellm
11
+
12
+ # Create an interaction with a model
13
+ response = litellm.interactions.create(
14
+ model="gemini-2.5-flash",
15
+ input="Hello, how are you?"
16
+ )
17
+
18
+ # Create an interaction with an agent
19
+ response = litellm.interactions.create(
20
+ agent="deep-research-pro-preview-12-2025",
21
+ input="Research the current state of cancer research"
22
+ )
23
+
24
+ # Async version
25
+ response = await litellm.interactions.acreate(...)
26
+
27
+ # Get an interaction
28
+ response = litellm.interactions.get(interaction_id="...")
29
+
30
+ # Delete an interaction
31
+ result = litellm.interactions.delete(interaction_id="...")
32
+ """
33
+
34
+ import asyncio
35
+ import contextvars
36
+ from functools import partial
37
+ from typing import Any, AsyncIterator, Coroutine, Dict, Iterator, List, Optional, Union
38
+
39
+ import httpx
40
+
41
+ import litellm
42
+ from litellm.interactions.http_handler import interactions_http_handler
43
+ from litellm.interactions.utils import (
44
+ InteractionsAPIRequestUtils,
45
+ get_provider_interactions_api_config,
46
+ )
47
+ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
48
+ from litellm.types.interactions import (
49
+ CancelInteractionResult,
50
+ DeleteInteractionResult,
51
+ InteractionInput,
52
+ InteractionsAPIResponse,
53
+ InteractionsAPIStreamingResponse,
54
+ InteractionTool,
55
+ )
56
+ from litellm.types.router import GenericLiteLLMParams
57
+ from litellm.utils import client
58
+
59
+ # ============================================================
60
+ # SDK Methods - CREATE INTERACTION
61
+ # ============================================================
62
+
63
+
64
+ @client
65
+ async def acreate(
66
+ # Model or Agent (one required per OpenAPI spec)
67
+ model: Optional[str] = None,
68
+ agent: Optional[str] = None,
69
+ # Input (required)
70
+ input: Optional[InteractionInput] = None,
71
+ # Tools (for model interactions)
72
+ tools: Optional[List[InteractionTool]] = None,
73
+ # System instruction
74
+ system_instruction: Optional[str] = None,
75
+ # Generation config
76
+ generation_config: Optional[Dict[str, Any]] = None,
77
+ # Streaming
78
+ stream: Optional[bool] = None,
79
+ # Storage
80
+ store: Optional[bool] = None,
81
+ # Background execution
82
+ background: Optional[bool] = None,
83
+ # Response format
84
+ response_modalities: Optional[List[str]] = None,
85
+ response_format: Optional[Dict[str, Any]] = None,
86
+ response_mime_type: Optional[str] = None,
87
+ # Continuation
88
+ previous_interaction_id: Optional[str] = None,
89
+ # Extra params
90
+ extra_headers: Optional[Dict[str, Any]] = None,
91
+ extra_body: Optional[Dict[str, Any]] = None,
92
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
93
+ # LiteLLM params
94
+ custom_llm_provider: Optional[str] = None,
95
+ **kwargs,
96
+ ) -> Union[InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]]:
97
+ """
98
+ Async: Create a new interaction using Google's Interactions API.
99
+
100
+ Per OpenAPI spec, provide either `model` or `agent`.
101
+
102
+ Args:
103
+ model: The model to use (e.g., "gemini-2.5-flash")
104
+ agent: The agent to use (e.g., "deep-research-pro-preview-12-2025")
105
+ input: The input content (string, content object, or list)
106
+ tools: Tools available for the model
107
+ system_instruction: System instruction for the interaction
108
+ generation_config: Generation configuration
109
+ stream: Whether to stream the response
110
+ store: Whether to store the response for later retrieval
111
+ background: Whether to run in background
112
+ response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO)
113
+ response_format: JSON schema for response format
114
+ response_mime_type: MIME type of the response
115
+ previous_interaction_id: ID of previous interaction for continuation
116
+ extra_headers: Additional headers
117
+ extra_body: Additional body parameters
118
+ timeout: Request timeout
119
+ custom_llm_provider: Override the LLM provider
120
+
121
+ Returns:
122
+ InteractionsAPIResponse or async iterator for streaming
123
+ """
124
+ local_vars = locals()
125
+ try:
126
+ loop = asyncio.get_event_loop()
127
+ kwargs["acreate_interaction"] = True
128
+
129
+ if custom_llm_provider is None and model:
130
+ _, custom_llm_provider, _, _ = litellm.get_llm_provider(
131
+ model=model, api_base=kwargs.get("api_base", None)
132
+ )
133
+ elif custom_llm_provider is None:
134
+ custom_llm_provider = "gemini"
135
+
136
+ func = partial(
137
+ create,
138
+ model=model,
139
+ agent=agent,
140
+ input=input,
141
+ tools=tools,
142
+ system_instruction=system_instruction,
143
+ generation_config=generation_config,
144
+ stream=stream,
145
+ store=store,
146
+ background=background,
147
+ response_modalities=response_modalities,
148
+ response_format=response_format,
149
+ response_mime_type=response_mime_type,
150
+ previous_interaction_id=previous_interaction_id,
151
+ extra_headers=extra_headers,
152
+ extra_body=extra_body,
153
+ timeout=timeout,
154
+ custom_llm_provider=custom_llm_provider,
155
+ **kwargs,
156
+ )
157
+
158
+ ctx = contextvars.copy_context()
159
+ func_with_context = partial(ctx.run, func)
160
+ init_response = await loop.run_in_executor(None, func_with_context)
161
+
162
+ if asyncio.iscoroutine(init_response):
163
+ response = await init_response
164
+ else:
165
+ response = init_response
166
+
167
+ return response # type: ignore
168
+ except Exception as e:
169
+ raise litellm.exception_type(
170
+ model=model,
171
+ custom_llm_provider=custom_llm_provider,
172
+ original_exception=e,
173
+ completion_kwargs=local_vars,
174
+ extra_kwargs=kwargs,
175
+ )
176
+
177
+
178
+ @client
179
+ def create(
180
+ # Model or Agent (one required per OpenAPI spec)
181
+ model: Optional[str] = None,
182
+ agent: Optional[str] = None,
183
+ # Input (required)
184
+ input: Optional[InteractionInput] = None,
185
+ # Tools (for model interactions)
186
+ tools: Optional[List[InteractionTool]] = None,
187
+ # System instruction
188
+ system_instruction: Optional[str] = None,
189
+ # Generation config
190
+ generation_config: Optional[Dict[str, Any]] = None,
191
+ # Streaming
192
+ stream: Optional[bool] = None,
193
+ # Storage
194
+ store: Optional[bool] = None,
195
+ # Background execution
196
+ background: Optional[bool] = None,
197
+ # Response format
198
+ response_modalities: Optional[List[str]] = None,
199
+ response_format: Optional[Dict[str, Any]] = None,
200
+ response_mime_type: Optional[str] = None,
201
+ # Continuation
202
+ previous_interaction_id: Optional[str] = None,
203
+ # Extra params
204
+ extra_headers: Optional[Dict[str, Any]] = None,
205
+ extra_body: Optional[Dict[str, Any]] = None,
206
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
207
+ # LiteLLM params
208
+ custom_llm_provider: Optional[str] = None,
209
+ **kwargs,
210
+ ) -> Union[
211
+ InteractionsAPIResponse,
212
+ Iterator[InteractionsAPIStreamingResponse],
213
+ Coroutine[
214
+ Any,
215
+ Any,
216
+ Union[InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]],
217
+ ],
218
+ ]:
219
+ """
220
+ Sync: Create a new interaction using Google's Interactions API.
221
+
222
+ Per OpenAPI spec, provide either `model` or `agent`.
223
+
224
+ Args:
225
+ model: The model to use (e.g., "gemini-2.5-flash")
226
+ agent: The agent to use (e.g., "deep-research-pro-preview-12-2025")
227
+ input: The input content (string, content object, or list)
228
+ tools: Tools available for the model
229
+ system_instruction: System instruction for the interaction
230
+ generation_config: Generation configuration
231
+ stream: Whether to stream the response
232
+ store: Whether to store the response for later retrieval
233
+ background: Whether to run in background
234
+ response_modalities: Requested response modalities (TEXT, IMAGE, AUDIO)
235
+ response_format: JSON schema for response format
236
+ response_mime_type: MIME type of the response
237
+ previous_interaction_id: ID of previous interaction for continuation
238
+ extra_headers: Additional headers
239
+ extra_body: Additional body parameters
240
+ timeout: Request timeout
241
+ custom_llm_provider: Override the LLM provider
242
+
243
+ Returns:
244
+ InteractionsAPIResponse or iterator for streaming
245
+ """
246
+ local_vars = locals()
247
+
248
+ try:
249
+ litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
250
+ litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
251
+ _is_async = kwargs.pop("acreate_interaction", False) is True
252
+
253
+ litellm_params = GenericLiteLLMParams(**kwargs)
254
+
255
+ if model:
256
+ model, custom_llm_provider, _, _ = litellm.get_llm_provider(
257
+ model=model,
258
+ custom_llm_provider=custom_llm_provider,
259
+ api_base=litellm_params.api_base,
260
+ api_key=litellm_params.api_key,
261
+ )
262
+ else:
263
+ custom_llm_provider = custom_llm_provider or "gemini"
264
+
265
+ interactions_api_config = get_provider_interactions_api_config(
266
+ provider=custom_llm_provider,
267
+ model=model,
268
+ )
269
+
270
+ # Get optional params using utility (similar to responses API pattern)
271
+ local_vars.update(kwargs)
272
+ optional_params = (
273
+ InteractionsAPIRequestUtils.get_requested_interactions_api_optional_params(
274
+ local_vars
275
+ )
276
+ )
277
+
278
+ # Check if this is a bridge provider (litellm_responses) - similar to responses API
279
+ # Either provider is explicitly "litellm_responses" or no config found (bridge to responses)
280
+ if (
281
+ custom_llm_provider == "litellm_responses"
282
+ or interactions_api_config is None
283
+ ):
284
+ # Bridge to litellm.responses() for non-native providers
285
+ from litellm.interactions.litellm_responses_transformation.handler import (
286
+ LiteLLMResponsesInteractionsHandler,
287
+ )
288
+
289
+ handler = LiteLLMResponsesInteractionsHandler()
290
+ return handler.interactions_api_handler(
291
+ model=model or "",
292
+ input=input,
293
+ optional_params=optional_params,
294
+ custom_llm_provider=custom_llm_provider,
295
+ _is_async=_is_async,
296
+ stream=stream,
297
+ **kwargs,
298
+ )
299
+
300
+ litellm_logging_obj.update_from_kwargs(
301
+ kwargs=kwargs,
302
+ model=model,
303
+ optional_params=dict(optional_params),
304
+ litellm_params={"litellm_call_id": litellm_call_id},
305
+ custom_llm_provider=custom_llm_provider,
306
+ )
307
+
308
+ response = interactions_http_handler.create_interaction(
309
+ model=model,
310
+ agent=agent,
311
+ input=input,
312
+ interactions_api_config=interactions_api_config,
313
+ optional_params=optional_params,
314
+ custom_llm_provider=custom_llm_provider,
315
+ litellm_params=litellm_params,
316
+ logging_obj=litellm_logging_obj,
317
+ extra_headers=extra_headers,
318
+ extra_body=extra_body,
319
+ timeout=timeout,
320
+ _is_async=_is_async,
321
+ stream=stream,
322
+ )
323
+
324
+ return response
325
+ except Exception as e:
326
+ raise litellm.exception_type(
327
+ model=model,
328
+ custom_llm_provider=custom_llm_provider,
329
+ original_exception=e,
330
+ completion_kwargs=local_vars,
331
+ extra_kwargs=kwargs,
332
+ )
333
+
334
+
335
+ # ============================================================
336
+ # SDK Methods - GET INTERACTION
337
+ # ============================================================
338
+
339
+
340
+ @client
341
+ async def aget(
342
+ interaction_id: str,
343
+ extra_headers: Optional[Dict[str, Any]] = None,
344
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
345
+ custom_llm_provider: Optional[str] = None,
346
+ **kwargs,
347
+ ) -> InteractionsAPIResponse:
348
+ """Async: Get an interaction by its ID."""
349
+ local_vars = locals()
350
+ try:
351
+ loop = asyncio.get_event_loop()
352
+ kwargs["aget_interaction"] = True
353
+
354
+ func = partial(
355
+ get,
356
+ interaction_id=interaction_id,
357
+ extra_headers=extra_headers,
358
+ timeout=timeout,
359
+ custom_llm_provider=custom_llm_provider or "gemini",
360
+ **kwargs,
361
+ )
362
+
363
+ ctx = contextvars.copy_context()
364
+ func_with_context = partial(ctx.run, func)
365
+ init_response = await loop.run_in_executor(None, func_with_context)
366
+
367
+ if asyncio.iscoroutine(init_response):
368
+ response = await init_response
369
+ else:
370
+ response = init_response
371
+
372
+ return response # type: ignore
373
+ except Exception as e:
374
+ raise litellm.exception_type(
375
+ model=None,
376
+ custom_llm_provider=custom_llm_provider or "gemini",
377
+ original_exception=e,
378
+ completion_kwargs=local_vars,
379
+ extra_kwargs=kwargs,
380
+ )
381
+
382
+
383
+ @client
384
+ def get(
385
+ interaction_id: str,
386
+ extra_headers: Optional[Dict[str, Any]] = None,
387
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
388
+ custom_llm_provider: Optional[str] = None,
389
+ **kwargs,
390
+ ) -> Union[InteractionsAPIResponse, Coroutine[Any, Any, InteractionsAPIResponse]]:
391
+ """Sync: Get an interaction by its ID."""
392
+ local_vars = locals()
393
+ custom_llm_provider = custom_llm_provider or "gemini"
394
+
395
+ try:
396
+ litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
397
+ litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
398
+ _is_async = kwargs.pop("aget_interaction", False) is True
399
+
400
+ litellm_params = GenericLiteLLMParams(**kwargs)
401
+
402
+ interactions_api_config = get_provider_interactions_api_config(
403
+ provider=custom_llm_provider,
404
+ )
405
+
406
+ if interactions_api_config is None:
407
+ raise ValueError(
408
+ f"Interactions API not supported for: {custom_llm_provider}"
409
+ )
410
+
411
+ litellm_logging_obj.update_from_kwargs(
412
+ kwargs=kwargs,
413
+ model=None,
414
+ optional_params={"interaction_id": interaction_id},
415
+ litellm_params={"litellm_call_id": litellm_call_id},
416
+ custom_llm_provider=custom_llm_provider,
417
+ )
418
+
419
+ return interactions_http_handler.get_interaction(
420
+ interaction_id=interaction_id,
421
+ interactions_api_config=interactions_api_config,
422
+ custom_llm_provider=custom_llm_provider,
423
+ litellm_params=litellm_params,
424
+ logging_obj=litellm_logging_obj,
425
+ extra_headers=extra_headers,
426
+ timeout=timeout,
427
+ _is_async=_is_async,
428
+ )
429
+ except Exception as e:
430
+ raise litellm.exception_type(
431
+ model=None,
432
+ custom_llm_provider=custom_llm_provider,
433
+ original_exception=e,
434
+ completion_kwargs=local_vars,
435
+ extra_kwargs=kwargs,
436
+ )
437
+
438
+
439
+ # ============================================================
440
+ # SDK Methods - DELETE INTERACTION
441
+ # ============================================================
442
+
443
+
444
+ @client
445
+ async def adelete(
446
+ interaction_id: str,
447
+ extra_headers: Optional[Dict[str, Any]] = None,
448
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
449
+ custom_llm_provider: Optional[str] = None,
450
+ **kwargs,
451
+ ) -> DeleteInteractionResult:
452
+ """Async: Delete an interaction by its ID."""
453
+ local_vars = locals()
454
+ try:
455
+ loop = asyncio.get_event_loop()
456
+ kwargs["adelete_interaction"] = True
457
+
458
+ func = partial(
459
+ delete,
460
+ interaction_id=interaction_id,
461
+ extra_headers=extra_headers,
462
+ timeout=timeout,
463
+ custom_llm_provider=custom_llm_provider or "gemini",
464
+ **kwargs,
465
+ )
466
+
467
+ ctx = contextvars.copy_context()
468
+ func_with_context = partial(ctx.run, func)
469
+ init_response = await loop.run_in_executor(None, func_with_context)
470
+
471
+ if asyncio.iscoroutine(init_response):
472
+ response = await init_response
473
+ else:
474
+ response = init_response
475
+
476
+ return response # type: ignore
477
+ except Exception as e:
478
+ raise litellm.exception_type(
479
+ model=None,
480
+ custom_llm_provider=custom_llm_provider or "gemini",
481
+ original_exception=e,
482
+ completion_kwargs=local_vars,
483
+ extra_kwargs=kwargs,
484
+ )
485
+
486
+
487
+ @client
488
+ def delete(
489
+ interaction_id: str,
490
+ extra_headers: Optional[Dict[str, Any]] = None,
491
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
492
+ custom_llm_provider: Optional[str] = None,
493
+ **kwargs,
494
+ ) -> Union[DeleteInteractionResult, Coroutine[Any, Any, DeleteInteractionResult]]:
495
+ """Sync: Delete an interaction by its ID."""
496
+ local_vars = locals()
497
+ custom_llm_provider = custom_llm_provider or "gemini"
498
+
499
+ try:
500
+ litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
501
+ litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
502
+ _is_async = kwargs.pop("adelete_interaction", False) is True
503
+
504
+ litellm_params = GenericLiteLLMParams(**kwargs)
505
+
506
+ interactions_api_config = get_provider_interactions_api_config(
507
+ provider=custom_llm_provider,
508
+ )
509
+
510
+ if interactions_api_config is None:
511
+ raise ValueError(
512
+ f"Interactions API not supported for: {custom_llm_provider}"
513
+ )
514
+
515
+ litellm_logging_obj.update_from_kwargs(
516
+ kwargs=kwargs,
517
+ model=None,
518
+ optional_params={"interaction_id": interaction_id},
519
+ litellm_params={"litellm_call_id": litellm_call_id},
520
+ custom_llm_provider=custom_llm_provider,
521
+ )
522
+
523
+ return interactions_http_handler.delete_interaction(
524
+ interaction_id=interaction_id,
525
+ interactions_api_config=interactions_api_config,
526
+ custom_llm_provider=custom_llm_provider,
527
+ litellm_params=litellm_params,
528
+ logging_obj=litellm_logging_obj,
529
+ extra_headers=extra_headers,
530
+ timeout=timeout,
531
+ _is_async=_is_async,
532
+ )
533
+ except Exception as e:
534
+ raise litellm.exception_type(
535
+ model=None,
536
+ custom_llm_provider=custom_llm_provider,
537
+ original_exception=e,
538
+ completion_kwargs=local_vars,
539
+ extra_kwargs=kwargs,
540
+ )
541
+
542
+
543
+ # ============================================================
544
+ # SDK Methods - CANCEL INTERACTION
545
+ # ============================================================
546
+
547
+
548
+ @client
549
+ async def acancel(
550
+ interaction_id: str,
551
+ extra_headers: Optional[Dict[str, Any]] = None,
552
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
553
+ custom_llm_provider: Optional[str] = None,
554
+ **kwargs,
555
+ ) -> CancelInteractionResult:
556
+ """Async: Cancel an interaction by its ID."""
557
+ local_vars = locals()
558
+ try:
559
+ loop = asyncio.get_event_loop()
560
+ kwargs["acancel_interaction"] = True
561
+
562
+ func = partial(
563
+ cancel,
564
+ interaction_id=interaction_id,
565
+ extra_headers=extra_headers,
566
+ timeout=timeout,
567
+ custom_llm_provider=custom_llm_provider or "gemini",
568
+ **kwargs,
569
+ )
570
+
571
+ ctx = contextvars.copy_context()
572
+ func_with_context = partial(ctx.run, func)
573
+ init_response = await loop.run_in_executor(None, func_with_context)
574
+
575
+ if asyncio.iscoroutine(init_response):
576
+ response = await init_response
577
+ else:
578
+ response = init_response
579
+
580
+ return response # type: ignore
581
+ except Exception as e:
582
+ raise litellm.exception_type(
583
+ model=None,
584
+ custom_llm_provider=custom_llm_provider or "gemini",
585
+ original_exception=e,
586
+ completion_kwargs=local_vars,
587
+ extra_kwargs=kwargs,
588
+ )
589
+
590
+
591
+ @client
592
+ def cancel(
593
+ interaction_id: str,
594
+ extra_headers: Optional[Dict[str, Any]] = None,
595
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
596
+ custom_llm_provider: Optional[str] = None,
597
+ **kwargs,
598
+ ) -> Union[CancelInteractionResult, Coroutine[Any, Any, CancelInteractionResult]]:
599
+ """Sync: Cancel an interaction by its ID."""
600
+ local_vars = locals()
601
+ custom_llm_provider = custom_llm_provider or "gemini"
602
+
603
+ try:
604
+ litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
605
+ litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
606
+ _is_async = kwargs.pop("acancel_interaction", False) is True
607
+
608
+ litellm_params = GenericLiteLLMParams(**kwargs)
609
+
610
+ interactions_api_config = get_provider_interactions_api_config(
611
+ provider=custom_llm_provider,
612
+ )
613
+
614
+ if interactions_api_config is None:
615
+ raise ValueError(
616
+ f"Interactions API not supported for: {custom_llm_provider}"
617
+ )
618
+
619
+ litellm_logging_obj.update_from_kwargs(
620
+ kwargs=kwargs,
621
+ model=None,
622
+ optional_params={"interaction_id": interaction_id},
623
+ litellm_params={"litellm_call_id": litellm_call_id},
624
+ custom_llm_provider=custom_llm_provider,
625
+ )
626
+
627
+ return interactions_http_handler.cancel_interaction(
628
+ interaction_id=interaction_id,
629
+ interactions_api_config=interactions_api_config,
630
+ custom_llm_provider=custom_llm_provider,
631
+ litellm_params=litellm_params,
632
+ logging_obj=litellm_logging_obj,
633
+ extra_headers=extra_headers,
634
+ timeout=timeout,
635
+ _is_async=_is_async,
636
+ )
637
+ except Exception as e:
638
+ raise litellm.exception_type(
639
+ model=None,
640
+ custom_llm_provider=custom_llm_provider,
641
+ original_exception=e,
642
+ completion_kwargs=local_vars,
643
+ extra_kwargs=kwargs,
644
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/streaming_iterator.py ADDED
@@ -0,0 +1,271 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Streaming iterators for the Interactions API.
3
+
4
+ This module provides streaming iterators that properly stream SSE responses
5
+ from the Google Interactions API, similar to the responses API streaming iterator.
6
+ """
7
+
8
+ import asyncio
9
+ import json
10
+ from datetime import datetime
11
+ from typing import Any, Dict, Optional
12
+
13
+ import httpx
14
+
15
+ from litellm._logging import verbose_logger
16
+ from litellm.constants import STREAM_SSE_DONE_STRING
17
+ from litellm.litellm_core_utils.asyncify import run_async_function
18
+ from litellm.litellm_core_utils.core_helpers import process_response_headers
19
+ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
20
+ from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
21
+ from litellm.litellm_core_utils.thread_pool_executor import executor
22
+ from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
23
+ from litellm.types.interactions import (
24
+ InteractionsAPIStreamingResponse,
25
+ )
26
+ from litellm.utils import CustomStreamWrapper
27
+
28
+
29
+ class BaseInteractionsAPIStreamingIterator:
30
+ """
31
+ Base class for streaming iterators that process responses from the Interactions API.
32
+
33
+ This class contains shared logic for both synchronous and asynchronous iterators.
34
+ """
35
+
36
+ def __init__(
37
+ self,
38
+ response: httpx.Response,
39
+ model: Optional[str],
40
+ interactions_api_config: BaseInteractionsAPIConfig,
41
+ logging_obj: LiteLLMLoggingObj,
42
+ litellm_metadata: Optional[Dict[str, Any]] = None,
43
+ custom_llm_provider: Optional[str] = None,
44
+ ):
45
+ self.response = response
46
+ self.model = model
47
+ self.logging_obj = logging_obj
48
+ self.finished = False
49
+ self.interactions_api_config = interactions_api_config
50
+ self.completed_response: Optional[InteractionsAPIStreamingResponse] = None
51
+ self.start_time = datetime.now()
52
+
53
+ # set request kwargs
54
+ self.litellm_metadata = litellm_metadata
55
+ self.custom_llm_provider = custom_llm_provider
56
+
57
+ # set hidden params for response headers
58
+ _api_base = get_api_base(
59
+ model=model or "",
60
+ optional_params=self.logging_obj.model_call_details.get(
61
+ "litellm_params", {}
62
+ ),
63
+ )
64
+ _model_info: Dict = (
65
+ litellm_metadata.get("model_info", {}) if litellm_metadata else {}
66
+ )
67
+ self._hidden_params = {
68
+ "model_id": _model_info.get("id", None),
69
+ "api_base": _api_base,
70
+ }
71
+ self._hidden_params["additional_headers"] = process_response_headers(
72
+ self.response.headers or {}
73
+ )
74
+
75
+ def _process_chunk(self, chunk: str) -> Optional[InteractionsAPIStreamingResponse]:
76
+ """Process a single chunk of data from the stream."""
77
+ if not chunk:
78
+ return None
79
+
80
+ # Handle SSE format (data: {...})
81
+ stripped_chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk)
82
+ if stripped_chunk is None:
83
+ return None
84
+
85
+ # Handle "[DONE]" marker
86
+ if stripped_chunk == STREAM_SSE_DONE_STRING:
87
+ self.finished = True
88
+ return None
89
+
90
+ try:
91
+ # Parse the JSON chunk
92
+ parsed_chunk = json.loads(stripped_chunk)
93
+
94
+ # Format as InteractionsAPIStreamingResponse
95
+ if isinstance(parsed_chunk, dict):
96
+ streaming_response = (
97
+ self.interactions_api_config.transform_streaming_response(
98
+ model=self.model,
99
+ parsed_chunk=parsed_chunk,
100
+ logging_obj=self.logging_obj,
101
+ )
102
+ )
103
+
104
+ # Store the completed response (check for status=completed)
105
+ if (
106
+ streaming_response
107
+ and getattr(streaming_response, "status", None) == "completed"
108
+ ):
109
+ self.completed_response = streaming_response
110
+ self._handle_logging_completed_response()
111
+
112
+ return streaming_response
113
+
114
+ return None
115
+ except json.JSONDecodeError:
116
+ # If we can't parse the chunk, continue
117
+ verbose_logger.debug(
118
+ f"Failed to parse streaming chunk: {stripped_chunk[:200]}..."
119
+ )
120
+ return None
121
+
122
+ def _handle_logging_completed_response(self):
123
+ """Base implementation - should be overridden by subclasses."""
124
+ pass
125
+
126
+
127
+ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
128
+ """
129
+ Async iterator for processing streaming responses from the Interactions API.
130
+ """
131
+
132
+ def __init__(
133
+ self,
134
+ response: httpx.Response,
135
+ model: Optional[str],
136
+ interactions_api_config: BaseInteractionsAPIConfig,
137
+ logging_obj: LiteLLMLoggingObj,
138
+ litellm_metadata: Optional[Dict[str, Any]] = None,
139
+ custom_llm_provider: Optional[str] = None,
140
+ ):
141
+ super().__init__(
142
+ response=response,
143
+ model=model,
144
+ interactions_api_config=interactions_api_config,
145
+ logging_obj=logging_obj,
146
+ litellm_metadata=litellm_metadata,
147
+ custom_llm_provider=custom_llm_provider,
148
+ )
149
+ self.stream_iterator = response.aiter_lines()
150
+
151
+ def __aiter__(self):
152
+ return self
153
+
154
+ async def __anext__(self) -> InteractionsAPIStreamingResponse:
155
+ try:
156
+ while True:
157
+ # Get the next chunk from the stream
158
+ try:
159
+ chunk = await self.stream_iterator.__anext__()
160
+ except StopAsyncIteration:
161
+ self.finished = True
162
+ raise StopAsyncIteration
163
+
164
+ result = self._process_chunk(chunk)
165
+
166
+ if self.finished:
167
+ raise StopAsyncIteration
168
+ elif result is not None:
169
+ return result
170
+ # If result is None, continue the loop to get the next chunk
171
+
172
+ except httpx.HTTPError as e:
173
+ # Handle HTTP errors
174
+ self.finished = True
175
+ raise e
176
+
177
+ def _handle_logging_completed_response(self):
178
+ """Handle logging for completed responses in async context."""
179
+ import copy
180
+
181
+ logging_response = copy.deepcopy(self.completed_response)
182
+
183
+ asyncio.create_task(
184
+ self.logging_obj.async_success_handler(
185
+ result=logging_response,
186
+ start_time=self.start_time,
187
+ end_time=datetime.now(),
188
+ cache_hit=None,
189
+ )
190
+ )
191
+
192
+ executor.submit(
193
+ self.logging_obj.success_handler,
194
+ result=logging_response,
195
+ cache_hit=None,
196
+ start_time=self.start_time,
197
+ end_time=datetime.now(),
198
+ )
199
+
200
+
201
+ class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
202
+ """
203
+ Synchronous iterator for processing streaming responses from the Interactions API.
204
+ """
205
+
206
+ def __init__(
207
+ self,
208
+ response: httpx.Response,
209
+ model: Optional[str],
210
+ interactions_api_config: BaseInteractionsAPIConfig,
211
+ logging_obj: LiteLLMLoggingObj,
212
+ litellm_metadata: Optional[Dict[str, Any]] = None,
213
+ custom_llm_provider: Optional[str] = None,
214
+ ):
215
+ super().__init__(
216
+ response=response,
217
+ model=model,
218
+ interactions_api_config=interactions_api_config,
219
+ logging_obj=logging_obj,
220
+ litellm_metadata=litellm_metadata,
221
+ custom_llm_provider=custom_llm_provider,
222
+ )
223
+ self.stream_iterator = response.iter_lines()
224
+
225
+ def __iter__(self):
226
+ return self
227
+
228
+ def __next__(self) -> InteractionsAPIStreamingResponse:
229
+ try:
230
+ while True:
231
+ # Get the next chunk from the stream
232
+ try:
233
+ chunk = next(self.stream_iterator)
234
+ except StopIteration:
235
+ self.finished = True
236
+ raise StopIteration
237
+
238
+ result = self._process_chunk(chunk)
239
+
240
+ if self.finished:
241
+ raise StopIteration
242
+ elif result is not None:
243
+ return result
244
+ # If result is None, continue the loop to get the next chunk
245
+
246
+ except httpx.HTTPError as e:
247
+ # Handle HTTP errors
248
+ self.finished = True
249
+ raise e
250
+
251
+ def _handle_logging_completed_response(self):
252
+ """Handle logging for completed responses in sync context."""
253
+ import copy
254
+
255
+ logging_response = copy.deepcopy(self.completed_response)
256
+
257
+ run_async_function(
258
+ async_function=self.logging_obj.async_success_handler,
259
+ result=logging_response,
260
+ start_time=self.start_time,
261
+ end_time=datetime.now(),
262
+ cache_hit=None,
263
+ )
264
+
265
+ executor.submit(
266
+ self.logging_obj.success_handler,
267
+ result=logging_response,
268
+ cache_hit=None,
269
+ start_time=self.start_time,
270
+ end_time=datetime.now(),
271
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/interactions/utils.py ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Utility functions for Interactions API.
3
+ """
4
+
5
+ from typing import Any, Dict, Optional, cast
6
+
7
+ from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
8
+ from litellm.types.interactions import InteractionsAPIOptionalRequestParams
9
+
10
+ # Valid optional parameter keys per OpenAPI spec
11
+ INTERACTIONS_API_OPTIONAL_PARAMS = {
12
+ "tools",
13
+ "system_instruction",
14
+ "generation_config",
15
+ "stream",
16
+ "store",
17
+ "background",
18
+ "response_modalities",
19
+ "response_format",
20
+ "response_mime_type",
21
+ "previous_interaction_id",
22
+ "agent_config",
23
+ }
24
+
25
+
26
+ def get_provider_interactions_api_config(
27
+ provider: str,
28
+ model: Optional[str] = None,
29
+ ) -> Optional[BaseInteractionsAPIConfig]:
30
+ """
31
+ Get the interactions API config for the given provider.
32
+
33
+ Args:
34
+ provider: The LLM provider name
35
+ model: Optional model name
36
+
37
+ Returns:
38
+ The provider-specific interactions API config, or None if not supported
39
+ """
40
+ from litellm.types.utils import LlmProviders
41
+
42
+ if provider == LlmProviders.GEMINI.value or provider == "gemini":
43
+ from litellm.llms.gemini.interactions.transformation import (
44
+ GoogleAIStudioInteractionsConfig,
45
+ )
46
+
47
+ return GoogleAIStudioInteractionsConfig()
48
+
49
+ return None
50
+
51
+
52
+ class InteractionsAPIRequestUtils:
53
+ """Helper utils for constructing Interactions API requests."""
54
+
55
+ @staticmethod
56
+ def get_requested_interactions_api_optional_params(
57
+ params: Dict[str, Any],
58
+ ) -> InteractionsAPIOptionalRequestParams:
59
+ """
60
+ Filter parameters to only include valid optional params per OpenAPI spec.
61
+
62
+ Args:
63
+ params: Dictionary of parameters to filter (typically from locals())
64
+
65
+ Returns:
66
+ Dict with only the valid optional parameters
67
+ """
68
+ from litellm.utils import PreProcessNonDefaultParams
69
+
70
+ custom_llm_provider = params.pop("custom_llm_provider", None)
71
+ special_params = params.pop("kwargs", {})
72
+ additional_drop_params = params.pop("additional_drop_params", None)
73
+
74
+ non_default_params = (
75
+ PreProcessNonDefaultParams.base_pre_process_non_default_params(
76
+ passed_params=params,
77
+ special_params=special_params,
78
+ custom_llm_provider=custom_llm_provider,
79
+ additional_drop_params=additional_drop_params,
80
+ default_param_values={
81
+ k: None for k in INTERACTIONS_API_OPTIONAL_PARAMS
82
+ },
83
+ additional_endpoint_specific_params=["input", "model", "agent"],
84
+ )
85
+ )
86
+
87
+ return cast(InteractionsAPIOptionalRequestParams, non_default_params)
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/README.md ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ## Folder Contents
2
+
3
+ This folder contains general-purpose utilities that are used in multiple places in the codebase.
4
+
5
+ Core files:
6
+ - `streaming_handler.py`: The core streaming logic + streaming related helper utils
7
+ - `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`.
8
+ - `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types.
9
+ - `default_encoding.py`: code for loading the default encoding (tiktoken)
10
+ - `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name.
11
+ - `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
12
+ - `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
13
+
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/api_route_to_call_types.py ADDED
@@ -0,0 +1,69 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Dictionary mapping API routes to their corresponding CallTypes in LiteLLM.
3
+
4
+ This dictionary maps each API endpoint to the CallTypes that can be used for that route.
5
+ Each route can have both async (prefixed with 'a') and sync call types.
6
+
7
+ Route patterns may contain placeholders like {agent_id}, {model}, {batch_id}; these
8
+ match a single path segment when resolving call types for a concrete path.
9
+ """
10
+
11
+ from typing import List, Optional
12
+
13
+ from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes
14
+
15
+
16
+ def _route_matches_pattern(route: str, pattern: str) -> bool:
17
+ """
18
+ Return True if the concrete route matches the pattern.
19
+ Pattern segments like {param} match any single path segment.
20
+ """
21
+ route_parts = route.strip("/").split("/")
22
+ pattern_parts = pattern.strip("/").split("/")
23
+ if len(route_parts) != len(pattern_parts):
24
+ return False
25
+ for r, p in zip(route_parts, pattern_parts):
26
+ if p.startswith("{") and p.endswith("}"):
27
+ continue
28
+ if r != p:
29
+ return False
30
+ return True
31
+
32
+
33
+ def get_call_types_for_route(route: str) -> Optional[List[CallTypes]]:
34
+ """
35
+ Get the list of CallTypes for a given API route.
36
+
37
+ Supports both exact keys and dynamic patterns (e.g. /a2a/my-agent/message/send
38
+ matches /a2a/{agent_id}/message/send).
39
+
40
+ Args:
41
+ route: API route path (e.g., "/chat/completions" or "/a2a/my-pydantic-agent/message/send")
42
+
43
+ Returns:
44
+ List of CallTypes for that route, or None if route not found
45
+ """
46
+ exact = API_ROUTE_TO_CALL_TYPES.get(route, None)
47
+ if exact is not None:
48
+ return exact
49
+ for pattern, call_types in API_ROUTE_TO_CALL_TYPES.items():
50
+ if _route_matches_pattern(route, pattern):
51
+ return call_types
52
+ return None
53
+
54
+
55
+ def get_routes_for_call_type(call_type: CallTypes) -> list:
56
+ """
57
+ Get all routes that use a specific CallType.
58
+
59
+ Args:
60
+ call_type: The CallType to search for
61
+
62
+ Returns:
63
+ List of routes that use this CallType
64
+ """
65
+ routes = []
66
+ for route, types in API_ROUTE_TO_CALL_TYPES.items():
67
+ if call_type in types:
68
+ routes.append(route)
69
+ return routes
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/app_crypto.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import base64
2
+ import json
3
+ import os
4
+ from typing import Optional
5
+
6
+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
7
+
8
+
9
+ class AppCrypto:
10
+ def __init__(self, master_key: bytes):
11
+ if len(master_key) != 32:
12
+ raise ValueError("Master key must be 32 bytes for AES-256-GCM")
13
+ self.key = master_key
14
+
15
+ def encrypt_json(self, data: dict, aad: Optional[bytes] = None) -> dict:
16
+ aes = AESGCM(self.key)
17
+ nonce = os.urandom(12)
18
+ plaintext = json.dumps(data).encode("utf-8")
19
+ ct = aes.encrypt(nonce, plaintext, aad)
20
+ ciphertext, tag = ct[:-16], ct[-16:]
21
+ return {
22
+ "nonce": base64.b64encode(nonce).decode(),
23
+ "ciphertext": base64.b64encode(ciphertext).decode(),
24
+ "tag": base64.b64encode(tag).decode(),
25
+ }
26
+
27
+ def decrypt_json(self, enc: dict, aad: Optional[bytes] = None) -> dict:
28
+ aes = AESGCM(self.key)
29
+ nonce = base64.b64decode(enc["nonce"])
30
+ ct = base64.b64decode(enc["ciphertext"])
31
+ tag = base64.b64decode(enc["tag"])
32
+ data = aes.decrypt(nonce, ct + tag, aad)
33
+ return json.loads(data.decode())
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/asyncify.py ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ import functools
3
+ from typing import Awaitable, Callable, Optional
4
+
5
+ import anyio
6
+ import anyio.to_thread
7
+ from typing_extensions import ParamSpec, TypeVar
8
+
9
+ T_ParamSpec = ParamSpec("T_ParamSpec")
10
+ T_Retval = TypeVar("T_Retval")
11
+
12
+
13
+ def function_has_argument(function: Callable, arg_name: str) -> bool:
14
+ """Helper function to check if a function has a specific argument."""
15
+ import inspect
16
+
17
+ signature = inspect.signature(function)
18
+ return arg_name in signature.parameters
19
+
20
+
21
+ def asyncify(
22
+ function: Callable[T_ParamSpec, T_Retval],
23
+ *,
24
+ cancellable: bool = False,
25
+ limiter: Optional[anyio.CapacityLimiter] = None,
26
+ ) -> Callable[T_ParamSpec, Awaitable[T_Retval]]:
27
+ """
28
+ Take a blocking function and create an async one that receives the same
29
+ positional and keyword arguments, and that when called, calls the original function
30
+ in a worker thread using `anyio.to_thread.run_sync()`.
31
+
32
+ If the `cancellable` option is enabled and the task waiting for its completion is
33
+ cancelled, the thread will still run its course but its return value (or any raised
34
+ exception) will be ignored.
35
+
36
+ ## Arguments
37
+ - `function`: a blocking regular callable (e.g. a function)
38
+ - `cancellable`: `True` to allow cancellation of the operation
39
+ - `limiter`: capacity limiter to use to limit the total amount of threads running
40
+ (if omitted, the default limiter is used)
41
+
42
+ ## Return
43
+ An async function that takes the same positional and keyword arguments as the
44
+ original one, that when called runs the same original function in a thread worker
45
+ and returns the result.
46
+ """
47
+
48
+ async def wrapper(
49
+ *args: T_ParamSpec.args, **kwargs: T_ParamSpec.kwargs
50
+ ) -> T_Retval:
51
+ partial_f = functools.partial(function, *args, **kwargs)
52
+
53
+ # In `v4.1.0` anyio added the `abandon_on_cancel` argument and deprecated the old
54
+ # `cancellable` argument, so we need to use the new `abandon_on_cancel` to avoid
55
+ # surfacing deprecation warnings.
56
+ if function_has_argument(anyio.to_thread.run_sync, "abandon_on_cancel"):
57
+ return await anyio.to_thread.run_sync(
58
+ partial_f,
59
+ abandon_on_cancel=cancellable,
60
+ limiter=limiter,
61
+ )
62
+
63
+ return await anyio.to_thread.run_sync(
64
+ partial_f,
65
+ cancellable=cancellable,
66
+ limiter=limiter,
67
+ )
68
+
69
+ return wrapper
70
+
71
+
72
+ def run_async_function(async_function, *args, **kwargs):
73
+ """
74
+ Helper utility to run an async function in a sync context.
75
+ Handles the case where there is an existing event loop running.
76
+
77
+ Args:
78
+ async_function (Callable): The async function to run
79
+ *args: Positional arguments to pass to the async function
80
+ **kwargs: Keyword arguments to pass to the async function
81
+
82
+ Returns:
83
+ The result of the async function execution
84
+
85
+ Example:
86
+ ```python
87
+ async def my_async_func(x, y):
88
+ return x + y
89
+
90
+ result = run_async_function(my_async_func, 1, 2)
91
+ ```
92
+ """
93
+ from concurrent.futures import ThreadPoolExecutor
94
+
95
+ def run_in_new_loop():
96
+ """Run the coroutine in a new event loop within this thread."""
97
+ new_loop = asyncio.new_event_loop()
98
+ try:
99
+ asyncio.set_event_loop(new_loop)
100
+ return new_loop.run_until_complete(async_function(*args, **kwargs))
101
+ finally:
102
+ new_loop.close()
103
+ asyncio.set_event_loop(None)
104
+
105
+ try:
106
+ # First, try to get the current event loop
107
+ _ = asyncio.get_running_loop()
108
+ # If we're already in an event loop, run in a separate thread
109
+ # to avoid nested event loop issues
110
+ with ThreadPoolExecutor(max_workers=1) as executor:
111
+ future = executor.submit(run_in_new_loop)
112
+ return future.result()
113
+
114
+ except RuntimeError:
115
+ # No running event loop, we can safely run in this thread
116
+ return run_in_new_loop()
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/audio_utils/audio_health_check.wav ADDED
Binary file (29.2 kB). View file
 
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/audio_utils/utils.py ADDED
@@ -0,0 +1,275 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Utils used for litellm.transcription() and litellm.atranscription()
3
+ """
4
+
5
+ import hashlib
6
+ import os
7
+ from dataclasses import dataclass
8
+ from typing import Optional
9
+
10
+ from litellm.types.files import get_file_mime_type_from_extension
11
+ from litellm.types.utils import FileTypes
12
+
13
+
14
+ @dataclass
15
+ class ProcessedAudioFile:
16
+ """
17
+ Processed audio file data.
18
+
19
+ Attributes:
20
+ file_content: The binary content of the audio file
21
+ filename: The filename (extracted or generated)
22
+ content_type: The MIME type of the audio file
23
+ """
24
+
25
+ file_content: bytes
26
+ filename: str
27
+ content_type: str
28
+
29
+
30
+ def process_audio_file(audio_file: FileTypes) -> ProcessedAudioFile:
31
+ """
32
+ Common utility function to process audio files for audio transcription APIs.
33
+
34
+ Handles various input types:
35
+ - File paths (str, os.PathLike)
36
+ - Raw bytes/bytearray
37
+ - Tuples (filename, content, optional content_type)
38
+ - File-like objects with read() method
39
+
40
+ Args:
41
+ audio_file: The audio file input in various formats
42
+
43
+ Returns:
44
+ ProcessedAudioFile: Structured data with file content, filename, and content type
45
+
46
+ Raises:
47
+ ValueError: If audio_file type is unsupported or content cannot be extracted
48
+ """
49
+ file_content = None
50
+ filename = None
51
+
52
+ if isinstance(audio_file, (bytes, bytearray)):
53
+ # Raw bytes
54
+ filename = "audio.wav"
55
+ file_content = bytes(audio_file)
56
+ elif isinstance(audio_file, (str, os.PathLike)):
57
+ # File path or PathLike
58
+ file_path = str(audio_file)
59
+ with open(file_path, "rb") as f:
60
+ file_content = f.read()
61
+ filename = file_path.split("/")[-1]
62
+ elif isinstance(audio_file, tuple):
63
+ # Tuple format: (filename, content, content_type) or (filename, content)
64
+ if len(audio_file) >= 2:
65
+ filename = audio_file[0] or "audio.wav"
66
+ content = audio_file[1]
67
+ if isinstance(content, (bytes, bytearray)):
68
+ file_content = bytes(content)
69
+ elif isinstance(content, (str, os.PathLike)):
70
+ # File path or PathLike
71
+ with open(str(content), "rb") as f:
72
+ file_content = f.read()
73
+ elif hasattr(content, "read"):
74
+ # File-like object
75
+ file_content = content.read()
76
+ if hasattr(content, "seek"):
77
+ content.seek(0)
78
+ else:
79
+ raise ValueError(f"Unsupported content type in tuple: {type(content)}")
80
+ else:
81
+ raise ValueError("Tuple must have at least 2 elements: (filename, content)")
82
+ elif hasattr(audio_file, "read") and not isinstance(
83
+ audio_file, (str, bytes, bytearray, tuple, os.PathLike)
84
+ ):
85
+ # File-like object (IO) - check this after all other types
86
+ filename = getattr(audio_file, "name", "audio.wav")
87
+ file_content = audio_file.read() # type: ignore
88
+ # Reset file pointer if possible
89
+ if hasattr(audio_file, "seek"):
90
+ audio_file.seek(0) # type: ignore
91
+ else:
92
+ raise ValueError(f"Unsupported audio_file type: {type(audio_file)}")
93
+
94
+ if file_content is None:
95
+ raise ValueError("Could not extract file content from audio_file")
96
+
97
+ # Determine content type using LiteLLM's file type utilities
98
+ content_type = "audio/wav" # Default fallback
99
+ if filename:
100
+ try:
101
+ # Extract extension from filename
102
+ extension = filename.split(".")[-1].lower() if "." in filename else "wav"
103
+ content_type = get_file_mime_type_from_extension(extension)
104
+ except ValueError:
105
+ # If extension is not recognized, fallback to audio/wav
106
+ content_type = "audio/wav"
107
+
108
+ return ProcessedAudioFile(
109
+ file_content=file_content, filename=filename, content_type=content_type
110
+ )
111
+
112
+
113
+ def get_audio_file_name(file_obj: FileTypes) -> str:
114
+ """
115
+ Safely get the name of a file-like object or return its string representation.
116
+
117
+ Args:
118
+ file_obj (Any): A file-like object or any other object.
119
+
120
+ Returns:
121
+ str: The name of the file if available, otherwise a string representation of the object.
122
+ """
123
+ if hasattr(file_obj, "name"):
124
+ return getattr(file_obj, "name")
125
+ elif hasattr(file_obj, "__str__"):
126
+ return str(file_obj)
127
+ else:
128
+ return repr(file_obj)
129
+
130
+
131
+ def get_audio_file_content_hash(file_obj: FileTypes) -> str:
132
+ """
133
+ Compute SHA-256 hash of audio file content for cache keys.
134
+ Falls back to filename hash if content extraction fails.
135
+ """
136
+ file_content: Optional[bytes] = None
137
+ fallback_filename: Optional[str] = None
138
+
139
+ if isinstance(file_obj, tuple):
140
+ if len(file_obj) < 2:
141
+ fallback_filename = str(file_obj[0]) if len(file_obj) > 0 else None
142
+ else:
143
+ fallback_filename = str(file_obj[0]) if file_obj[0] is not None else None
144
+ file_content_obj = file_obj[1]
145
+ else:
146
+ file_content_obj = file_obj
147
+ fallback_filename = get_audio_file_name(file_obj)
148
+
149
+ try:
150
+ if isinstance(file_content_obj, (bytes, bytearray)):
151
+ file_content = bytes(file_content_obj)
152
+ elif isinstance(file_content_obj, (str, os.PathLike)):
153
+ try:
154
+ with open(str(file_content_obj), "rb") as f:
155
+ file_content = f.read()
156
+ if fallback_filename is None:
157
+ fallback_filename = str(file_content_obj)
158
+ except (OSError, IOError):
159
+ fallback_filename = str(file_content_obj)
160
+ file_content = None
161
+ elif hasattr(file_content_obj, "read"):
162
+ try:
163
+ current_position = (
164
+ file_content_obj.tell()
165
+ if hasattr(file_content_obj, "tell")
166
+ else None
167
+ )
168
+ if hasattr(file_content_obj, "seek"):
169
+ file_content_obj.seek(0)
170
+ file_content = file_content_obj.read() # type: ignore
171
+ if current_position is not None and hasattr(file_content_obj, "seek"):
172
+ file_content_obj.seek(current_position) # type: ignore
173
+ except (OSError, IOError, AttributeError):
174
+ file_content = None
175
+ else:
176
+ file_content = None
177
+ except Exception:
178
+ file_content = None
179
+
180
+ if file_content is not None and isinstance(file_content, bytes):
181
+ try:
182
+ hash_object = hashlib.sha256(file_content)
183
+ return hash_object.hexdigest()
184
+ except Exception:
185
+ pass
186
+
187
+ if fallback_filename:
188
+ hash_object = hashlib.sha256(fallback_filename.encode("utf-8"))
189
+ return hash_object.hexdigest()
190
+
191
+ file_obj_str = str(file_obj)
192
+ hash_object = hashlib.sha256(file_obj_str.encode("utf-8"))
193
+ return hash_object.hexdigest()
194
+
195
+
196
+ def get_audio_file_for_health_check() -> FileTypes:
197
+ """
198
+ Get an audio file for health check
199
+
200
+ Returns the content of `audio_health_check.wav` in the same directory as this file
201
+ """
202
+ pwd = os.path.dirname(os.path.realpath(__file__))
203
+ file_path = os.path.join(pwd, "audio_health_check.wav")
204
+ return open(file_path, "rb")
205
+
206
+
207
+ def calculate_request_duration(file: FileTypes) -> Optional[float]:
208
+ """
209
+ Calculate audio duration from file content.
210
+
211
+ Args:
212
+ file: The audio file (can be file path, bytes, or file-like object)
213
+
214
+ Returns:
215
+ Duration in seconds, or None if extraction fails or soundfile is not available
216
+ """
217
+ try:
218
+ import soundfile as sf
219
+ except ImportError:
220
+ # soundfile not available, cannot extract duration
221
+ return None
222
+
223
+ try:
224
+ import io
225
+
226
+ # Handle different file input types
227
+ file_content: Optional[bytes] = None
228
+
229
+ if isinstance(file, (bytes, bytearray)):
230
+ # Raw bytes
231
+ file_content = bytes(file)
232
+ elif isinstance(file, (str, os.PathLike)):
233
+ # File path
234
+ with open(str(file), "rb") as f:
235
+ file_content = f.read()
236
+ elif isinstance(file, tuple):
237
+ # Tuple format: (filename, content, optional content_type)
238
+ if len(file) >= 2:
239
+ content = file[1]
240
+ if isinstance(content, bytes):
241
+ file_content = content
242
+ elif hasattr(content, "read") and not isinstance(
243
+ content, (str, os.PathLike)
244
+ ):
245
+ # File-like object in tuple
246
+ current_pos = getattr(content, "tell", lambda: None)()
247
+ # Seek to start to ensure we read the entire content
248
+ if hasattr(content, "seek"):
249
+ content.seek(0)
250
+ file_content = content.read()
251
+ if current_pos is not None and hasattr(content, "seek"):
252
+ content.seek(current_pos)
253
+ elif hasattr(file, "read") and not isinstance(file, tuple):
254
+ # File-like object (including BytesIO)
255
+ current_position = file.tell() if hasattr(file, "tell") else None
256
+ # Seek to start to ensure we read the entire content
257
+ if hasattr(file, "seek"):
258
+ file.seek(0)
259
+ file_content = file.read()
260
+ # Reset file position if possible
261
+ if current_position is not None and hasattr(file, "seek"):
262
+ file.seek(current_position)
263
+
264
+ if file_content is None or not isinstance(file_content, bytes):
265
+ return None
266
+
267
+ # Extract duration using soundfile
268
+ file_object = io.BytesIO(file_content)
269
+ with sf.SoundFile(file_object) as audio:
270
+ duration = len(audio) / audio.samplerate
271
+ return duration
272
+
273
+ except Exception:
274
+ # Silently fail if duration extraction fails
275
+ return None
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/cached_imports.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Cached imports module for LiteLLM.
3
+
4
+ This module provides cached import functionality to avoid repeated imports
5
+ inside functions that are critical to performance.
6
+ """
7
+
8
+ from typing import TYPE_CHECKING, Callable, Optional, Type
9
+
10
+ # Type annotations for cached imports
11
+ if TYPE_CHECKING:
12
+ from litellm.litellm_core_utils.litellm_logging import Logging
13
+ from litellm.litellm_core_utils.coroutine_checker import CoroutineChecker
14
+
15
+ # Global cache variables
16
+ _LiteLLMLogging: Optional[Type["Logging"]] = None
17
+ _coroutine_checker: Optional["CoroutineChecker"] = None
18
+ _set_callbacks: Optional[Callable] = None
19
+
20
+
21
+ def get_litellm_logging_class() -> Type["Logging"]:
22
+ """Get the cached LiteLLM Logging class, initializing if needed."""
23
+ global _LiteLLMLogging
24
+ if _LiteLLMLogging is not None:
25
+ return _LiteLLMLogging
26
+ from litellm.litellm_core_utils.litellm_logging import Logging
27
+
28
+ _LiteLLMLogging = Logging
29
+ return _LiteLLMLogging
30
+
31
+
32
+ def get_coroutine_checker() -> "CoroutineChecker":
33
+ """Get the cached coroutine checker instance, initializing if needed."""
34
+ global _coroutine_checker
35
+ if _coroutine_checker is not None:
36
+ return _coroutine_checker
37
+ from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
38
+
39
+ _coroutine_checker = coroutine_checker
40
+ return _coroutine_checker
41
+
42
+
43
+ def get_set_callbacks() -> Callable:
44
+ """Get the cached set_callbacks function, initializing if needed."""
45
+ global _set_callbacks
46
+ if _set_callbacks is not None:
47
+ return _set_callbacks
48
+ from litellm.litellm_core_utils.litellm_logging import set_callbacks
49
+
50
+ _set_callbacks = set_callbacks
51
+ return _set_callbacks
52
+
53
+
54
+ def clear_cached_imports() -> None:
55
+ """Clear all cached imports. Useful for testing or memory management."""
56
+ global _LiteLLMLogging, _coroutine_checker, _set_callbacks
57
+ _LiteLLMLogging = None
58
+ _coroutine_checker = None
59
+ _set_callbacks = None
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/cli_token_utils.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ CLI Token Utilities
3
+
4
+ SDK-level utilities for reading CLI authentication tokens.
5
+ This module has no dependencies on proxy code and can be safely imported at the SDK level.
6
+ """
7
+
8
+ import json
9
+ import os
10
+ from pathlib import Path
11
+ from typing import Optional
12
+
13
+
14
+ def get_cli_token_file_path() -> str:
15
+ """Get the path to the CLI token file"""
16
+ home_dir = Path.home()
17
+ config_dir = home_dir / ".litellm"
18
+ return str(config_dir / "token.json")
19
+
20
+
21
+ def load_cli_token() -> Optional[dict]:
22
+ """Load CLI token data from file"""
23
+ token_file = get_cli_token_file_path()
24
+ if not os.path.exists(token_file):
25
+ return None
26
+
27
+ try:
28
+ with open(token_file, "r") as f:
29
+ return json.load(f)
30
+ except (json.JSONDecodeError, IOError):
31
+ return None
32
+
33
+
34
+ def get_litellm_gateway_api_key() -> Optional[str]:
35
+ """
36
+ Get the stored CLI API key for use with LiteLLM SDK.
37
+
38
+ This function reads the token file created by `litellm-proxy login`
39
+ and returns the API key for use in Python scripts.
40
+
41
+ Returns:
42
+ str: The API key if found, None otherwise
43
+
44
+ Example:
45
+ >>> import litellm
46
+ >>> api_key = litellm.get_litellm_gateway_api_key()
47
+ >>> if api_key:
48
+ >>> response = litellm.completion(
49
+ >>> model="gpt-3.5-turbo",
50
+ >>> messages=[{"role": "user", "content": "Hello"}],
51
+ >>> api_key=api_key,
52
+ >>> base_url="https://your-proxy.com/v1"
53
+ >>> )
54
+ """
55
+ token_data = load_cli_token()
56
+ if token_data and "key" in token_data:
57
+ return token_data["key"]
58
+ return None
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/core_helpers.py ADDED
@@ -0,0 +1,437 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # What is this?
2
+ ## Helper utilities
3
+ from typing import TYPE_CHECKING, Any, Iterable, List, Literal, Optional, Union
4
+
5
+ import httpx
6
+
7
+ from litellm._logging import verbose_logger
8
+ from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
9
+
10
+ if TYPE_CHECKING:
11
+ from opentelemetry.trace import Span as _Span
12
+
13
+ from litellm.types.utils import ModelResponseStream
14
+
15
+ Span = Union[_Span, Any]
16
+ else:
17
+ Span = Any
18
+
19
+
20
+ def safe_divide_seconds(
21
+ seconds: float, denominator: float, default: Optional[float] = None
22
+ ) -> Optional[float]:
23
+ """
24
+ Safely divide seconds by denominator, handling zero division.
25
+
26
+ Args:
27
+ seconds: Time duration in seconds
28
+ denominator: The divisor (e.g., number of tokens)
29
+ default: Value to return if division by zero (defaults to None)
30
+
31
+ Returns:
32
+ The result of the division as a float (seconds per unit), or default if denominator is zero
33
+ """
34
+ if denominator <= 0:
35
+ return default
36
+
37
+ return float(seconds / denominator)
38
+
39
+
40
+ def safe_divide(
41
+ numerator: Union[int, float],
42
+ denominator: Union[int, float],
43
+ default: Union[int, float] = 0,
44
+ ) -> Union[int, float]:
45
+ """
46
+ Safely divide two numbers, returning a default value if denominator is zero.
47
+
48
+ Args:
49
+ numerator: The number to divide
50
+ denominator: The number to divide by
51
+ default: Value to return if denominator is zero (defaults to 0)
52
+
53
+ Returns:
54
+ The result of numerator/denominator, or default if denominator is zero
55
+ """
56
+ if denominator == 0:
57
+ return default
58
+ return numerator / denominator
59
+
60
+
61
+ _FINISH_REASON_MAP: dict[str, OpenAIChatCompletionFinishReason] = {
62
+ # Anthropic
63
+ "stop_sequence": "stop",
64
+ "end_turn": "stop",
65
+ "max_tokens": "length",
66
+ "tool_use": "tool_calls",
67
+ "refusal": "content_filter",
68
+ "compaction": "length",
69
+ # Cohere
70
+ "COMPLETE": "stop",
71
+ "ERROR_TOXIC": "content_filter",
72
+ "ERROR": "stop",
73
+ # HuggingFace / Together AI
74
+ "eos_token": "stop",
75
+ "eos": "stop",
76
+ # Gemini / Vertex AI
77
+ "STOP": "stop",
78
+ "MAX_TOKENS": "length",
79
+ "SAFETY": "content_filter",
80
+ "RECITATION": "content_filter",
81
+ "FINISH_REASON_UNSPECIFIED": "stop",
82
+ "MALFORMED_FUNCTION_CALL": "stop",
83
+ "LANGUAGE": "content_filter",
84
+ "OTHER": "content_filter",
85
+ "BLOCKLIST": "content_filter",
86
+ "PROHIBITED_CONTENT": "content_filter",
87
+ "SPII": "content_filter",
88
+ "IMAGE_SAFETY": "content_filter",
89
+ "IMAGE_PROHIBITED_CONTENT": "content_filter",
90
+ "TOO_MANY_TOOL_CALLS": "stop",
91
+ "MALFORMED_RESPONSE": "stop",
92
+ # Bedrock
93
+ "guardrail_intervened": "content_filter",
94
+ # OpenAI passthrough
95
+ "stop": "stop",
96
+ "length": "length",
97
+ "tool_calls": "tool_calls",
98
+ "function_call": "function_call",
99
+ "content_filter": "content_filter",
100
+ # Anthropic Sonnet 4
101
+ "content_filtered": "content_filter",
102
+ }
103
+
104
+
105
+ def map_finish_reason(finish_reason: str) -> OpenAIChatCompletionFinishReason:
106
+ mapped = _FINISH_REASON_MAP.get(finish_reason)
107
+ if mapped is None:
108
+ verbose_logger.warning(
109
+ "Unmapped finish_reason '%s', defaulting to 'stop'", finish_reason
110
+ )
111
+ return "stop"
112
+ return mapped
113
+
114
+
115
+ def remove_index_from_tool_calls(
116
+ messages: Optional[List[AllMessageValues]],
117
+ ):
118
+ if messages is not None:
119
+ for message in messages:
120
+ _tool_calls = message.get("tool_calls")
121
+ if _tool_calls is not None and isinstance(_tool_calls, list):
122
+ for tool_call in _tool_calls:
123
+ if (
124
+ isinstance(tool_call, dict) and "index" in tool_call
125
+ ): # Type guard to ensure it's a dict
126
+ tool_call.pop("index", None)
127
+
128
+ return
129
+
130
+
131
+ def remove_items_at_indices(items: Optional[List[Any]], indices: Iterable[int]) -> None:
132
+ """Remove items from a list in-place by index"""
133
+ if items is None:
134
+ return
135
+ for index in sorted(set(indices), reverse=True):
136
+ if 0 <= index < len(items):
137
+ items.pop(index)
138
+
139
+
140
+ def add_missing_spend_metadata_to_litellm_metadata(
141
+ litellm_metadata: dict, metadata: dict
142
+ ) -> dict:
143
+ """
144
+ Helper to get litellm metadata for spend tracking
145
+
146
+ PATCH for issue where both `litellm_metadata` and `metadata` are present in the kwargs
147
+ and user_api_key values are in 'metadata'.
148
+ """
149
+ potential_spend_tracking_metadata_substring = "user_api_key"
150
+ for key, value in metadata.items():
151
+ if potential_spend_tracking_metadata_substring in key:
152
+ litellm_metadata[key] = value
153
+ return litellm_metadata
154
+
155
+
156
+ def get_metadata_variable_name_from_kwargs(
157
+ kwargs: dict,
158
+ ) -> Literal["metadata", "litellm_metadata"]:
159
+ """
160
+ Helper to return what the "metadata" field should be called in the request data
161
+
162
+ - New endpoints return `litellm_metadata`
163
+ - Old endpoints return `metadata`
164
+
165
+ Context:
166
+ - LiteLLM used `metadata` as an internal field for storing metadata
167
+ - OpenAI then started using this field for their metadata
168
+ - LiteLLM is now moving to using `litellm_metadata` for our metadata
169
+ """
170
+ return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
171
+
172
+
173
+ def get_litellm_metadata_from_kwargs(kwargs: dict):
174
+ """
175
+ Helper to get litellm metadata from all litellm request kwargs
176
+
177
+ Return `litellm_metadata` if it exists, otherwise return `metadata`
178
+ """
179
+ litellm_params = kwargs.get("litellm_params", {})
180
+ if litellm_params:
181
+ metadata = litellm_params.get("metadata", {})
182
+ litellm_metadata = litellm_params.get("litellm_metadata", {})
183
+ if litellm_metadata and metadata:
184
+ litellm_metadata = add_missing_spend_metadata_to_litellm_metadata(
185
+ litellm_metadata, metadata
186
+ )
187
+ if litellm_metadata:
188
+ return litellm_metadata
189
+ elif metadata:
190
+ return metadata
191
+
192
+ return {}
193
+
194
+
195
+ def reconstruct_model_name(
196
+ model_name: str,
197
+ custom_llm_provider: Optional[str],
198
+ metadata: dict,
199
+ ) -> str:
200
+ """Reconstruct full model name with provider prefix for logging."""
201
+ # Check if deployment model name from router metadata is available (has original prefix)
202
+ deployment_model_name = metadata.get("deployment")
203
+ if deployment_model_name and "/" in deployment_model_name:
204
+ # Use the deployment model name which preserves the original provider prefix
205
+ return deployment_model_name
206
+ elif custom_llm_provider and model_name and "/" not in model_name:
207
+ # Only add prefix for Bedrock (not for direct Anthropic API)
208
+ # This ensures Bedrock models get the prefix while direct Anthropic models don't
209
+ if custom_llm_provider == "bedrock":
210
+ return f"{custom_llm_provider}/{model_name}"
211
+ return model_name
212
+
213
+
214
+ # Helper functions used for OTEL logging
215
+ def _get_parent_otel_span_from_kwargs(
216
+ kwargs: Optional[dict] = None,
217
+ ) -> Union[Span, None]:
218
+ try:
219
+ if kwargs is None:
220
+ return None
221
+ litellm_params = kwargs.get("litellm_params")
222
+ _metadata = kwargs.get("metadata") or {}
223
+ if "litellm_parent_otel_span" in _metadata:
224
+ return _metadata["litellm_parent_otel_span"]
225
+ elif (
226
+ litellm_params is not None
227
+ and litellm_params.get("metadata") is not None
228
+ and "litellm_parent_otel_span" in litellm_params.get("metadata", {})
229
+ ):
230
+ return litellm_params["metadata"]["litellm_parent_otel_span"]
231
+ elif "litellm_parent_otel_span" in kwargs:
232
+ return kwargs["litellm_parent_otel_span"]
233
+ return None
234
+ except Exception as e:
235
+ verbose_logger.exception(
236
+ "Error in _get_parent_otel_span_from_kwargs: " + str(e)
237
+ )
238
+ return None
239
+
240
+
241
+ def process_response_headers(response_headers: Union[httpx.Headers, dict]) -> dict:
242
+ from litellm.types.utils import OPENAI_RESPONSE_HEADERS
243
+
244
+ openai_headers = {}
245
+ processed_headers = {}
246
+ additional_headers = {}
247
+
248
+ for k, v in response_headers.items():
249
+ if k in OPENAI_RESPONSE_HEADERS: # return openai-compatible headers
250
+ openai_headers[k] = v
251
+ if k.startswith(
252
+ "llm_provider-"
253
+ ): # return raw provider headers (incl. openai-compatible ones)
254
+ processed_headers[k] = v
255
+ else:
256
+ additional_headers["{}-{}".format("llm_provider", k)] = v
257
+
258
+ additional_headers = {
259
+ **openai_headers,
260
+ **processed_headers,
261
+ **additional_headers,
262
+ }
263
+ return additional_headers
264
+
265
+
266
+ def preserve_upstream_non_openai_attributes(
267
+ model_response: "ModelResponseStream", original_chunk: "ModelResponseStream"
268
+ ):
269
+ """
270
+ Preserve non-OpenAI attributes from the original chunk.
271
+ """
272
+ # Access model_fields on the class, not the instance, to avoid Pydantic 2.11+ deprecation warnings
273
+ expected_keys = set(type(model_response).model_fields.keys()).union({"usage"})
274
+ for key, value in original_chunk.model_dump().items():
275
+ if key not in expected_keys:
276
+ setattr(model_response, key, value)
277
+
278
+
279
+ def safe_deep_copy(data):
280
+ """
281
+ Safe Deep Copy
282
+
283
+ The LiteLLM request may contain objects that cannot be pickled/deep-copied
284
+ (e.g., tracing spans, locks, clients).
285
+
286
+ This helper deep-copies each top-level key independently; on failure keeps
287
+ original ref
288
+ """
289
+ import copy
290
+
291
+ import litellm
292
+
293
+ if litellm.safe_memory_mode is True:
294
+ return data
295
+
296
+ litellm_parent_otel_span: Optional[Any] = None
297
+ # Step 1: Remove the litellm_parent_otel_span
298
+ litellm_parent_otel_span = None
299
+ if isinstance(data, dict):
300
+ # remove litellm_parent_otel_span since this is not picklable
301
+ if "metadata" in data and "litellm_parent_otel_span" in data["metadata"]:
302
+ litellm_parent_otel_span = data["metadata"].pop("litellm_parent_otel_span")
303
+ data["metadata"]["litellm_parent_otel_span"] = "placeholder"
304
+ if (
305
+ "litellm_metadata" in data
306
+ and "litellm_parent_otel_span" in data["litellm_metadata"]
307
+ ):
308
+ litellm_parent_otel_span = data["litellm_metadata"].pop(
309
+ "litellm_parent_otel_span"
310
+ )
311
+ data["litellm_metadata"]["litellm_parent_otel_span"] = "placeholder"
312
+
313
+ # Step 2: Per-key deepcopy with fallback
314
+ if isinstance(data, dict):
315
+ new_data = {}
316
+ for k, v in data.items():
317
+ try:
318
+ new_data[k] = copy.deepcopy(v)
319
+ except Exception:
320
+ new_data[k] = v
321
+ else:
322
+ try:
323
+ new_data = copy.deepcopy(data)
324
+ except Exception:
325
+ new_data = data
326
+
327
+ # Step 3: re-add the litellm_parent_otel_span after doing a deep copy
328
+ if isinstance(data, dict) and litellm_parent_otel_span is not None:
329
+ if "metadata" in data and "litellm_parent_otel_span" in data["metadata"]:
330
+ data["metadata"]["litellm_parent_otel_span"] = litellm_parent_otel_span
331
+ if (
332
+ "litellm_metadata" in data
333
+ and "litellm_parent_otel_span" in data["litellm_metadata"]
334
+ ):
335
+ data["litellm_metadata"][
336
+ "litellm_parent_otel_span"
337
+ ] = litellm_parent_otel_span
338
+ return new_data
339
+
340
+
341
+ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
342
+ """
343
+ Recursively filter out Exception objects and callable objects from dicts/lists.
344
+
345
+ This is a defensive utility to prevent deepcopy failures when exception objects
346
+ are accidentally stored in parameter dictionaries (e.g., optional_params).
347
+ Also filters callable objects (functions) to prevent JSON serialization errors.
348
+ Exceptions and callables should not be stored in params - this function removes them.
349
+
350
+ Args:
351
+ data: The data structure to filter (dict, list, or any other type)
352
+ max_depth: Maximum recursion depth to prevent infinite loops
353
+
354
+ Returns:
355
+ Filtered data structure with Exception and callable objects removed, or None if the
356
+ entire input was an Exception or callable
357
+ """
358
+ if max_depth <= 0:
359
+ return data
360
+
361
+ # Skip exception objects
362
+ if isinstance(data, Exception):
363
+ return None
364
+ # Skip callable objects (functions, methods, lambdas) but not classes (type objects)
365
+ if callable(data) and not isinstance(data, type):
366
+ return None
367
+ # Skip known non-serializable object types (Logging, Router, etc.)
368
+ obj_type_name = type(data).__name__
369
+ if obj_type_name in ["Logging", "LiteLLMLoggingObj", "Router"]:
370
+ return None
371
+
372
+ if isinstance(data, dict):
373
+ result: dict[str, Any] = {}
374
+ for k, v in data.items():
375
+ # Skip exception and callable values
376
+ if isinstance(v, Exception) or (callable(v) and not isinstance(v, type)):
377
+ continue
378
+ try:
379
+ filtered = filter_exceptions_from_params(v, max_depth - 1)
380
+ if filtered is not None:
381
+ result[k] = filtered
382
+ except Exception:
383
+ # Skip values that cause errors during filtering
384
+ continue
385
+ return result
386
+ elif isinstance(data, list):
387
+ result_list: list[Any] = []
388
+ for item in data:
389
+ # Skip exception and callable items
390
+ if isinstance(item, Exception) or (
391
+ callable(item) and not isinstance(item, type)
392
+ ):
393
+ continue
394
+ try:
395
+ filtered = filter_exceptions_from_params(item, max_depth - 1)
396
+ if filtered is not None:
397
+ result_list.append(filtered)
398
+ except Exception:
399
+ # Skip items that cause errors during filtering
400
+ continue
401
+ return result_list
402
+ else:
403
+ return data
404
+
405
+
406
+ def filter_internal_params(
407
+ data: dict, additional_internal_params: Optional[set] = None
408
+ ) -> dict:
409
+ """
410
+ Filter out LiteLLM internal parameters that shouldn't be sent to provider APIs.
411
+
412
+ This removes internal/MCP-related parameters that are used by LiteLLM internally
413
+ but should not be included in API requests to providers.
414
+
415
+ Args:
416
+ data: Dictionary of parameters to filter
417
+ additional_internal_params: Optional set of additional internal parameter names to filter
418
+
419
+ Returns:
420
+ Filtered dictionary with internal parameters removed
421
+ """
422
+ if not isinstance(data, dict):
423
+ return data
424
+
425
+ # Known internal parameters that should never be sent to provider APIs
426
+ internal_params = {
427
+ "skip_mcp_handler",
428
+ "mcp_handler_context",
429
+ "_skip_mcp_handler",
430
+ }
431
+
432
+ # Add any additional internal params if provided
433
+ if additional_internal_params:
434
+ internal_params.update(additional_internal_params)
435
+
436
+ # Filter out internal parameters
437
+ return {k: v for k, v in data.items() if k not in internal_params}
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/coroutine_checker.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # CoroutineChecker utility for checking if functions/callables are coroutines or coroutine functions
2
+
3
+ import inspect
4
+ from typing import Any
5
+ from weakref import WeakKeyDictionary
6
+ from litellm.constants import (
7
+ COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY,
8
+ )
9
+
10
+
11
+ class CoroutineChecker:
12
+ """Utility class for checking coroutine status of functions and callables.
13
+
14
+ Simple bounded cache using WeakKeyDictionary to avoid memory leaks.
15
+ """
16
+
17
+ def __init__(self):
18
+ self._cache = WeakKeyDictionary()
19
+ self._max_size = COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY
20
+
21
+ def is_async_callable(self, callback: Any) -> bool:
22
+ """Fast, cached check for whether a callback is an async function.
23
+ Falls back gracefully if the object cannot be weak-referenced or cached.
24
+ 2.59x speedup.
25
+ """
26
+ # Fast path: check cache first (most common case)
27
+ try:
28
+ cached = self._cache.get(callback)
29
+ if cached is not None:
30
+ return cached
31
+ except Exception:
32
+ pass
33
+
34
+ # Determine target - optimized path for common cases
35
+ target = callback
36
+ if not inspect.isfunction(target) and not inspect.ismethod(target):
37
+ try:
38
+ call_attr = getattr(target, "__call__", None)
39
+ if call_attr is not None:
40
+ target = call_attr
41
+ except Exception:
42
+ pass
43
+
44
+ # Compute result
45
+ try:
46
+ result = inspect.iscoroutinefunction(target)
47
+ except Exception:
48
+ result = False
49
+
50
+ # Cache the result with size enforcement
51
+ try:
52
+ # Simple size enforcement: clear cache if it gets too large
53
+ if len(self._cache) >= self._max_size:
54
+ self._cache.clear()
55
+
56
+ self._cache[callback] = result
57
+ except Exception:
58
+ pass
59
+
60
+ return result
61
+
62
+
63
+ # Global instance for backward compatibility and convenience
64
+ coroutine_checker = CoroutineChecker()
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/credential_accessor.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Utils for accessing credentials."""
2
+
3
+ from typing import List
4
+
5
+ import litellm
6
+ from litellm.types.utils import CredentialItem
7
+
8
+
9
+ class CredentialAccessor:
10
+ @staticmethod
11
+ def get_credential_values(credential_name: str) -> dict:
12
+ """Safe accessor for credentials."""
13
+
14
+ if not litellm.credential_list:
15
+ return {}
16
+ for credential in litellm.credential_list:
17
+ if credential.credential_name == credential_name:
18
+ return credential.credential_values.copy()
19
+ return {}
20
+
21
+ @staticmethod
22
+ def upsert_credentials(credentials: List[CredentialItem]):
23
+ """Add a credential to the list of credentials."""
24
+
25
+ credential_names = [cred.credential_name for cred in litellm.credential_list]
26
+
27
+ for credential in credentials:
28
+ if credential.credential_name in credential_names:
29
+ # Find and replace the existing credential in the list
30
+ for i, existing_cred in enumerate(litellm.credential_list):
31
+ if existing_cred.credential_name == credential.credential_name:
32
+ litellm.credential_list[i] = credential
33
+ break
34
+ else:
35
+ litellm.credential_list.append(credential)
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/custom_logger_registry.py ADDED
@@ -0,0 +1,184 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Registry mapping the callback class string to the class type.
3
+
4
+ This is used to get the class type from the callback class string.
5
+
6
+ Example:
7
+ "datadog" -> DataDogLogger
8
+ "prometheus" -> PrometheusLogger
9
+ """
10
+
11
+ from typing import Union
12
+
13
+ from litellm import _custom_logger_compatible_callbacks_literal
14
+ from litellm.integrations.agentops import AgentOps
15
+ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
16
+ from litellm.integrations.argilla import ArgillaLogger
17
+ from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger
18
+ from litellm.integrations.bitbucket import BitBucketPromptManager
19
+ from litellm.integrations.braintrust_logging import BraintrustLogger
20
+ from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
21
+ from litellm.integrations.datadog.datadog import DataDogLogger
22
+ from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger
23
+ from litellm.integrations.datadog.datadog_metrics import DatadogMetricsLogger
24
+ from litellm.integrations.deepeval import DeepEvalLogger
25
+ from litellm.integrations.dotprompt import DotpromptManager
26
+ from litellm.integrations.focus.focus_logger import FocusLogger
27
+ from litellm.integrations.vantage.vantage_logger import VantageLogger
28
+ from litellm.integrations.galileo import GalileoObserve
29
+ from litellm.integrations.gcs_bucket.gcs_bucket import GCSBucketLogger
30
+ from litellm.integrations.gcs_pubsub.pub_sub import GcsPubSubLogger
31
+ from litellm.integrations.gitlab import GitLabPromptManager
32
+ from litellm.integrations.humanloop import HumanloopLogger
33
+ from litellm.integrations.lago import LagoLogger
34
+ from litellm.integrations.langfuse.langfuse_prompt_management import (
35
+ LangfusePromptManagement,
36
+ )
37
+ from litellm.integrations.langsmith import LangsmithLogger
38
+ from litellm.integrations.litellm_agent import LiteLLMAgentModelResolver
39
+ from litellm.integrations.literal_ai import LiteralAILogger
40
+ from litellm.integrations.mlflow import MlflowLogger
41
+ from litellm.integrations.openmeter import OpenMeterLogger
42
+ from litellm.integrations.opentelemetry import OpenTelemetry
43
+ from litellm.integrations.opik.opik import OpikLogger
44
+ from litellm.integrations.posthog import PostHogLogger
45
+ from litellm.integrations.prometheus import PrometheusLogger
46
+ from litellm.integrations.s3_v2 import S3Logger
47
+ from litellm.integrations.sqs import SQSLogger
48
+ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook import (
49
+ VectorStorePreCallHook,
50
+ )
51
+ from litellm.proxy.hooks.dynamic_rate_limiter import _PROXY_DynamicRateLimitHandler
52
+ from litellm.proxy.hooks.dynamic_rate_limiter_v3 import _PROXY_DynamicRateLimitHandlerV3
53
+
54
+
55
+ class CustomLoggerRegistry:
56
+ """
57
+ Registry mapping the callback class string to the class type.
58
+ """
59
+
60
+ CALLBACK_CLASS_STR_TO_CLASS_TYPE = {
61
+ "lago": LagoLogger,
62
+ "openmeter": OpenMeterLogger,
63
+ "braintrust": BraintrustLogger,
64
+ "galileo": GalileoObserve,
65
+ "langsmith": LangsmithLogger,
66
+ "literalai": LiteralAILogger,
67
+ "litellm_agent": LiteLLMAgentModelResolver,
68
+ "prometheus": PrometheusLogger,
69
+ "datadog": DataDogLogger,
70
+ "datadog_llm_observability": DataDogLLMObsLogger,
71
+ "datadog_metrics": DatadogMetricsLogger,
72
+ "gcs_bucket": GCSBucketLogger,
73
+ "opik": OpikLogger,
74
+ "argilla": ArgillaLogger,
75
+ "opentelemetry": OpenTelemetry,
76
+ "azure_storage": AzureBlobStorageLogger,
77
+ "humanloop": HumanloopLogger,
78
+ # OTEL compatible loggers
79
+ "logfire": OpenTelemetry,
80
+ "arize": OpenTelemetry,
81
+ "langfuse_otel": OpenTelemetry,
82
+ "arize_phoenix": OpenTelemetry,
83
+ "langtrace": OpenTelemetry,
84
+ "weave_otel": OpenTelemetry,
85
+ "levo": OpenTelemetry,
86
+ "mlflow": MlflowLogger,
87
+ "langfuse": LangfusePromptManagement,
88
+ "otel": OpenTelemetry,
89
+ "gcs_pubsub": GcsPubSubLogger,
90
+ "anthropic_cache_control_hook": AnthropicCacheControlHook,
91
+ "agentops": AgentOps,
92
+ "deepeval": DeepEvalLogger,
93
+ "s3_v2": S3Logger,
94
+ "aws_sqs": SQSLogger,
95
+ "dynamic_rate_limiter": _PROXY_DynamicRateLimitHandler,
96
+ "dynamic_rate_limiter_v3": _PROXY_DynamicRateLimitHandlerV3,
97
+ "vector_store_pre_call_hook": VectorStorePreCallHook,
98
+ "dotprompt": DotpromptManager,
99
+ "bitbucket": BitBucketPromptManager,
100
+ "gitlab": GitLabPromptManager,
101
+ "cloudzero": CloudZeroLogger,
102
+ "focus": FocusLogger,
103
+ "vantage": VantageLogger,
104
+ "posthog": PostHogLogger,
105
+ }
106
+
107
+ try:
108
+ from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import (
109
+ PagerDutyAlerting,
110
+ )
111
+ from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import (
112
+ ResendEmailLogger,
113
+ )
114
+ from litellm_enterprise.enterprise_callbacks.send_emails.sendgrid_email import (
115
+ SendGridEmailLogger,
116
+ )
117
+ from litellm_enterprise.enterprise_callbacks.send_emails.smtp_email import (
118
+ SMTPEmailLogger,
119
+ )
120
+
121
+ from litellm.integrations.generic_api.generic_api_callback import (
122
+ GenericAPILogger,
123
+ )
124
+
125
+ enterprise_loggers = {
126
+ "pagerduty": PagerDutyAlerting,
127
+ "generic_api": GenericAPILogger,
128
+ "resend_email": ResendEmailLogger,
129
+ "sendgrid_email": SendGridEmailLogger,
130
+ "smtp_email": SMTPEmailLogger,
131
+ }
132
+ CALLBACK_CLASS_STR_TO_CLASS_TYPE.update(enterprise_loggers)
133
+ except ImportError:
134
+ pass # enterprise not installed
135
+
136
+ @classmethod
137
+ def get_callback_str_from_class_type(cls, class_type: type) -> Union[str, None]:
138
+ """
139
+ Get the callback string from the class type.
140
+
141
+ Args:
142
+ class_type: The class type to find the string for
143
+
144
+ Returns:
145
+ str: The callback string, or None if not found
146
+ """
147
+ for (
148
+ callback_str,
149
+ callback_class,
150
+ ) in cls.CALLBACK_CLASS_STR_TO_CLASS_TYPE.items():
151
+ if callback_class == class_type:
152
+ return callback_str
153
+ return None
154
+
155
+ @classmethod
156
+ def get_all_callback_strs_from_class_type(cls, class_type: type) -> list[str]:
157
+ """
158
+ Get all callback strings that map to the same class type.
159
+ Some class types (like OpenTelemetry) have multiple string mappings.
160
+
161
+ Args:
162
+ class_type: The class type to find all strings for
163
+
164
+ Returns:
165
+ list: List of callback strings that map to the class type
166
+ """
167
+ callback_strs: list[str] = []
168
+ for (
169
+ callback_str,
170
+ callback_class,
171
+ ) in cls.CALLBACK_CLASS_STR_TO_CLASS_TYPE.items():
172
+ if callback_class == class_type:
173
+ callback_strs.append(callback_str)
174
+ return callback_strs
175
+
176
+ @classmethod
177
+ def get_class_type_for_custom_logger_name(
178
+ cls,
179
+ custom_logger_name: _custom_logger_compatible_callbacks_literal,
180
+ ) -> type:
181
+ """
182
+ Get the class type for a given custom logger name
183
+ """
184
+ return cls.CALLBACK_CLASS_STR_TO_CLASS_TYPE[custom_logger_name]
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/dd_tracing.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Handles Tracing on DataDog Traces.
3
+
4
+ If the ddtrace package is not installed, the tracer will be a no-op.
5
+ """
6
+
7
+ from contextlib import contextmanager
8
+ from typing import TYPE_CHECKING, Any, Optional, Union
9
+
10
+ from litellm.secret_managers.main import get_secret_bool
11
+
12
+ if TYPE_CHECKING:
13
+ from ddtrace.tracer import Tracer as DD_TRACER
14
+ else:
15
+ DD_TRACER = Any
16
+
17
+
18
+ class NullSpan:
19
+ """A no-op span implementation."""
20
+
21
+ def __enter__(self):
22
+ return self
23
+
24
+ def __exit__(self, *args):
25
+ pass
26
+
27
+ def finish(self):
28
+ pass
29
+
30
+
31
+ @contextmanager
32
+ def null_tracer(name, **kwargs):
33
+ """Context manager that yields a no-op span."""
34
+ yield NullSpan()
35
+
36
+
37
+ class NullTracer:
38
+ """A no-op tracer implementation."""
39
+
40
+ def trace(self, name, **kwargs):
41
+ return NullSpan()
42
+
43
+ def wrap(self, name=None, **kwargs):
44
+ # If called with no arguments (as @tracer.wrap())
45
+ if callable(name):
46
+ return name
47
+
48
+ # If called with arguments (as @tracer.wrap(name="something"))
49
+ def decorator(f):
50
+ return f
51
+
52
+ return decorator
53
+
54
+
55
+ def _should_use_dd_tracer():
56
+ """Returns True if `USE_DDTRACE` is set to True in .env"""
57
+ return get_secret_bool("USE_DDTRACE", False) is True
58
+
59
+
60
+ def _should_use_dd_profiler():
61
+ """Returns True if `USE_DDPROFILER` is set to True in .env"""
62
+ return get_secret_bool("USE_DDPROFILER", False) is True
63
+
64
+
65
+ # Initialize tracer
66
+ should_use_dd_tracer = _should_use_dd_tracer()
67
+ tracer: Union[NullTracer, DD_TRACER] = NullTracer()
68
+ # We need to ensure tracer is never None and always has the required methods
69
+ if should_use_dd_tracer:
70
+ try:
71
+ from ddtrace import tracer as dd_tracer
72
+
73
+ # Define the type to match what's expected by the code using this module
74
+ tracer = dd_tracer
75
+ except ImportError:
76
+ tracer = NullTracer()
77
+ else:
78
+ tracer = NullTracer()
79
+
80
+
81
+ def get_active_span() -> Optional[Any]:
82
+ """
83
+ Return the active Datadog span, checking current span first and then root span.
84
+ """
85
+ try:
86
+ current_span_fn = getattr(tracer, "current_span", None)
87
+ if callable(current_span_fn):
88
+ current_span = current_span_fn()
89
+ if current_span is not None:
90
+ return current_span
91
+
92
+ current_root_span_fn = getattr(tracer, "current_root_span", None)
93
+ if callable(current_root_span_fn):
94
+ return current_root_span_fn()
95
+ except Exception:
96
+ return None
97
+ return None
98
+
99
+
100
+ def set_active_span_tag(tag_key: str, tag_value: str) -> bool:
101
+ """
102
+ Best-effort helper to set a tag on the active Datadog span.
103
+
104
+ Returns:
105
+ bool: True if a span tag was set, False otherwise.
106
+ """
107
+ if not tag_key or tag_value is None:
108
+ return False
109
+
110
+ span = get_active_span()
111
+ if span is None:
112
+ return False
113
+
114
+ try:
115
+ if hasattr(span, "set_tag_str"):
116
+ span.set_tag_str(tag_key, str(tag_value))
117
+ return True
118
+ if hasattr(span, "set_tag"):
119
+ span.set_tag(tag_key, str(tag_value))
120
+ return True
121
+ except Exception:
122
+ return False
123
+ return False
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/default_encoding.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ import litellm
4
+
5
+ try:
6
+ # New and recommended way to access resources
7
+ from importlib import resources
8
+
9
+ filename = str(resources.files(litellm).joinpath("litellm_core_utils/tokenizers"))
10
+ except (ImportError, AttributeError):
11
+ # Old way to access resources, which setuptools deprecated some time ago
12
+ import pkg_resources # type: ignore
13
+
14
+ filename = pkg_resources.resource_filename(
15
+ __name__, "litellm_core_utils/tokenizers"
16
+ )
17
+
18
+ # Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory
19
+ # unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR.
20
+ # This keeps tiktoken fully offline-capable by default (see #1071).
21
+ custom_cache_dir = os.getenv("CUSTOM_TIKTOKEN_CACHE_DIR")
22
+ if custom_cache_dir:
23
+ # If the user opts into a custom cache dir, ensure it exists.
24
+ os.makedirs(custom_cache_dir, exist_ok=True)
25
+ cache_dir = custom_cache_dir
26
+ else:
27
+ cache_dir = filename
28
+
29
+ os.environ[
30
+ "TIKTOKEN_CACHE_DIR"
31
+ ] = cache_dir # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071
32
+
33
+ import tiktoken
34
+ import time
35
+ import random
36
+
37
+ # Retry logic to handle race conditions when multiple processes try to create
38
+ # the tiktoken cache file simultaneously (common in parallel test execution on Windows)
39
+ _max_retries = 5
40
+ _retry_delay = 0.1 # Start with 100ms
41
+
42
+ for attempt in range(_max_retries):
43
+ try:
44
+ encoding = tiktoken.get_encoding("cl100k_base")
45
+ break
46
+ except (FileExistsError, OSError):
47
+ if attempt == _max_retries - 1:
48
+ # Last attempt, re-raise the exception
49
+ raise
50
+ # Exponential backoff with jitter to reduce collision probability
51
+ delay = _retry_delay * (2**attempt) + random.uniform(0, 0.1)
52
+ time.sleep(delay)
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/dot_notation_indexing.py ADDED
@@ -0,0 +1,255 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Path-based navigation utilities for nested dictionaries.
3
+
4
+ This module provides utilities for reading and deleting values in nested
5
+ dictionaries using dot notation and JSONPath-like array syntax.
6
+
7
+ Custom implementation with zero external dependencies.
8
+
9
+ Supported syntax:
10
+ - "field" - top-level field
11
+ - "parent.child" - nested field
12
+ - "parent\\.with\\.dots.child" - keys containing dots (escape with backslash)
13
+ - "array[*]" - all array elements (wildcard)
14
+ - "array[0]" - specific array element (index)
15
+ - "array[*].field" - field in all array elements
16
+
17
+ Examples:
18
+ >>> data = {"tools": [{"name": "t1", "input_examples": ["ex"]}]}
19
+ >>> delete_nested_value(data, "tools[*].input_examples")
20
+ {"tools": [{"name": "t1"}]}
21
+
22
+ Used by JWT Auth to get the user role from the token, and by
23
+ additional_drop_params to remove nested fields from optional parameters.
24
+ """
25
+
26
+ from typing import Any, Dict, List, Optional, TypeVar, Union
27
+
28
+ T = TypeVar("T")
29
+
30
+
31
+ def get_nested_value(
32
+ data: Dict[str, Any], key_path: str, default: Optional[T] = None
33
+ ) -> Optional[T]:
34
+ """
35
+ Retrieves a value from a nested dictionary using dot notation.
36
+
37
+ Args:
38
+ data: The dictionary to search in
39
+ key_path: The path to the value using dot notation (e.g., "a.b.c")
40
+ default: The default value to return if the path is not found
41
+
42
+ Returns:
43
+ The value at the specified path, or the default value if not found
44
+
45
+ Example:
46
+ >>> data = {"a": {"b": {"c": "value"}}}
47
+ >>> get_nested_value(data, "a.b.c")
48
+ 'value'
49
+ >>> get_nested_value(data, "a.b.d", "default")
50
+ 'default'
51
+ >>> data = {"kubernetes.io": {"namespace": "default"}}
52
+ >>> get_nested_value(data, "kubernetes\\.io.namespace")
53
+ 'default'
54
+ """
55
+ if not key_path:
56
+ return default
57
+
58
+ # Remove metadata. prefix if it exists
59
+ key_path = (
60
+ key_path.replace("metadata.", "", 1)
61
+ if key_path.startswith("metadata.")
62
+ else key_path
63
+ )
64
+
65
+ # Split the key path into parts, respecting escaped dots (\.)
66
+ # Use a temporary placeholder, split on unescaped dots, then restore
67
+ placeholder = "\x00"
68
+ parts = key_path.replace("\\.", placeholder).split(".")
69
+ parts = [p.replace(placeholder, ".") for p in parts]
70
+
71
+ # Traverse through the dictionary
72
+ current: Any = data
73
+ for part in parts:
74
+ try:
75
+ current = current[part]
76
+ except (KeyError, TypeError):
77
+ return default
78
+
79
+ # If default is None, we can return any type
80
+ if default is None:
81
+ return current
82
+
83
+ # Otherwise, ensure the type matches the default
84
+ return current if isinstance(current, type(default)) else default
85
+
86
+
87
+ def _parse_path_segments(path: str) -> list:
88
+ """
89
+ Parse a JSONPath-like string into segments using regex.
90
+
91
+ Handles:
92
+ - Dot notation: "a.b.c" → ["a", "b", "c"]
93
+ - Array wildcards: "a[*].b" → ["a", "[*]", "b"]
94
+ - Array indices: "a[0].b" → ["a", "[0]", "b"]
95
+
96
+ Args:
97
+ path: JSONPath-like path string
98
+
99
+ Returns:
100
+ List of path segments
101
+
102
+ Example:
103
+ >>> _parse_path_segments("tools[*].arr[0].field")
104
+ ["tools", "[*]", "arr", "[0]", "field"]
105
+ """
106
+ import re
107
+
108
+ # Match field names OR bracket expressions
109
+ # Pattern: field_name (anything except . or [) | [anything_in_brackets]
110
+ pattern = r"[^\.\[]+|\[[^\]]*\]"
111
+ segments = re.findall(pattern, path)
112
+ return segments
113
+
114
+
115
+ def _delete_nested_value_custom(
116
+ data: Union[Dict[str, Any], List[Any]],
117
+ segments: list,
118
+ segment_index: int = 0,
119
+ ) -> None:
120
+ """
121
+ Recursively delete a field from nested data using parsed segments.
122
+
123
+ Modifies data in-place (caller must deep copy first).
124
+
125
+ Args:
126
+ data: Dictionary or list to modify
127
+ segments: Parsed path segments
128
+ segment_index: Current position in segments list
129
+ """
130
+ if segment_index >= len(segments):
131
+ return
132
+
133
+ segment = segments[segment_index]
134
+ is_last = segment_index == len(segments) - 1
135
+
136
+ # Handle array wildcard: [*]
137
+ if segment == "[*]":
138
+ if isinstance(data, list):
139
+ for item in data:
140
+ if is_last:
141
+ # Can't delete array elements themselves, skip
142
+ pass
143
+ else:
144
+ # Only recurse if item is a dict or list (nested structure)
145
+ if isinstance(item, (dict, list)):
146
+ _delete_nested_value_custom(item, segments, segment_index + 1)
147
+ return
148
+
149
+ # Handle array index: [0], [1], [2], etc.
150
+ if segment.startswith("[") and segment.endswith("]"):
151
+ try:
152
+ index = int(segment[1:-1])
153
+ if isinstance(data, list) and 0 <= index < len(data):
154
+ if is_last:
155
+ # Can't delete array elements themselves, skip
156
+ pass
157
+ else:
158
+ # Only recurse if element is a dict or list (nested structure)
159
+ element = data[index]
160
+ if isinstance(element, (dict, list)):
161
+ _delete_nested_value_custom(
162
+ element, segments, segment_index + 1
163
+ )
164
+ except (ValueError, IndexError):
165
+ # Invalid index, skip
166
+ pass
167
+ return
168
+
169
+ # Handle regular field navigation
170
+ if isinstance(data, dict):
171
+ if is_last:
172
+ # Delete the field
173
+ data.pop(segment, None)
174
+ else:
175
+ # Navigate deeper
176
+ if segment in data:
177
+ next_segment = (
178
+ segments[segment_index + 1]
179
+ if segment_index + 1 < len(segments)
180
+ else None
181
+ )
182
+
183
+ # If next segment is array notation, current field should be list
184
+ if next_segment and (next_segment.startswith("[")):
185
+ if isinstance(data[segment], list):
186
+ _delete_nested_value_custom(
187
+ data[segment], segments, segment_index + 1
188
+ )
189
+ # Otherwise navigate into dict
190
+ elif isinstance(data[segment], dict):
191
+ _delete_nested_value_custom(
192
+ data[segment], segments, segment_index + 1
193
+ )
194
+
195
+
196
+ def delete_nested_value(
197
+ data: Dict[str, Any],
198
+ path: str,
199
+ depth: int = 0,
200
+ max_depth: int = 20,
201
+ ) -> Dict[str, Any]:
202
+ """
203
+ Delete a field from nested data using JSONPath notation.
204
+
205
+ Custom implementation - no external dependencies.
206
+
207
+ Supports:
208
+ - "field" - top-level field
209
+ - "parent.child" - nested field
210
+ - "array[*]" - all array elements (wildcard)
211
+ - "array[0]" - specific array element (index)
212
+ - "array[*].field" - field in all array elements
213
+
214
+ Args:
215
+ data: Dictionary to modify (creates deep copy)
216
+ path: JSONPath-like path string
217
+ depth: Current recursion depth (kept for API compatibility)
218
+ max_depth: Maximum recursion depth (kept for API compatibility)
219
+
220
+ Returns:
221
+ New dictionary with field removed at path
222
+
223
+ Example:
224
+ >>> data = {"tools": [{"name": "t1", "input_examples": ["ex"]}]}
225
+ >>> delete_nested_value(data, "tools[*].input_examples")
226
+ {"tools": [{"name": "t1"}]}
227
+ """
228
+ import copy
229
+
230
+ result = copy.deepcopy(data)
231
+
232
+ try:
233
+ # Parse path into segments
234
+ segments = _parse_path_segments(path)
235
+
236
+ if not segments:
237
+ return result
238
+
239
+ # Delete using custom recursive implementation
240
+ _delete_nested_value_custom(result, segments, 0)
241
+
242
+ except Exception:
243
+ # Invalid path or parsing error - silently skip
244
+ pass
245
+
246
+ return result
247
+
248
+
249
+ def is_nested_path(path: str) -> bool:
250
+ """
251
+ Check if path requires nested handling.
252
+
253
+ Returns True if path contains '.' or '[' (array notation).
254
+ """
255
+ return "." in path or "[" in path
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/duration_parser.py ADDED
@@ -0,0 +1,391 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Helper utilities for parsing durations - 1s, 1d, 10d, 30d, 1mo, 2mo
3
+
4
+ duration_in_seconds is used in diff parts of the code base, example
5
+ - Router - Provider budget routing
6
+ - Proxy - Key, Team Generation
7
+ """
8
+
9
+ import re
10
+ import time
11
+ from datetime import datetime, timedelta, timezone, tzinfo
12
+ from typing import Optional, Tuple
13
+ from zoneinfo import ZoneInfo
14
+
15
+
16
+ def _extract_from_regex(duration: str) -> Tuple[int, str]:
17
+ match = re.match(r"(\d+)(mo|[smhdw]?)", duration)
18
+
19
+ if not match:
20
+ raise ValueError("Invalid duration format")
21
+
22
+ value, unit = match.groups()
23
+ value = int(value)
24
+
25
+ return value, unit
26
+
27
+
28
+ def get_last_day_of_month(year, month):
29
+ # Handle December case
30
+ if month == 12:
31
+ return 31
32
+ # Next month is January, so subtract a day from March 1st
33
+ next_month = datetime(year=year, month=month + 1, day=1)
34
+ last_day_of_month = (next_month - timedelta(days=1)).day
35
+ return last_day_of_month
36
+
37
+
38
+ def duration_in_seconds(duration: str) -> int:
39
+ """
40
+ Parameters:
41
+ - duration:
42
+ - "<number>s" - seconds
43
+ - "<number>m" - minutes
44
+ - "<number>h" - hours
45
+ - "<number>d" - days
46
+ - "<number>w" - weeks
47
+ - "<number>mo" - months
48
+
49
+ Returns time in seconds till when budget needs to be reset
50
+ """
51
+ value, unit = _extract_from_regex(duration=duration)
52
+
53
+ if unit == "s":
54
+ return value
55
+ elif unit == "m":
56
+ return value * 60
57
+ elif unit == "h":
58
+ return value * 3600
59
+ elif unit == "d":
60
+ return value * 86400
61
+ elif unit == "w":
62
+ return value * 604800
63
+ elif unit == "mo":
64
+ now = time.time()
65
+ current_time = datetime.fromtimestamp(now)
66
+
67
+ # Calculate target month and year, handling overflow past December
68
+ total_months = current_time.month - 1 + value # 0-indexed months
69
+ target_year = current_time.year + total_months // 12
70
+ target_month = total_months % 12 + 1 # back to 1-indexed
71
+
72
+ # Determine the day to set for next month
73
+ target_day = current_time.day
74
+ last_day_of_target_month = get_last_day_of_month(target_year, target_month)
75
+
76
+ if target_day > last_day_of_target_month:
77
+ target_day = last_day_of_target_month
78
+
79
+ next_month = datetime(
80
+ year=target_year,
81
+ month=target_month,
82
+ day=target_day,
83
+ hour=current_time.hour,
84
+ minute=current_time.minute,
85
+ second=current_time.second,
86
+ microsecond=current_time.microsecond,
87
+ )
88
+
89
+ # Calculate the duration until the first day of the next month
90
+ duration_until_next_month = next_month - current_time
91
+ return int(duration_until_next_month.total_seconds())
92
+
93
+ else:
94
+ raise ValueError(f"Unsupported duration unit, passed duration: {duration}")
95
+
96
+
97
+ def get_next_standardized_reset_time(
98
+ duration: str, current_time: datetime, timezone_str: str = "UTC"
99
+ ) -> datetime:
100
+ """
101
+ Get the next standardized reset time based on the duration.
102
+
103
+ All durations will reset at predictable intervals, aligned from the current time:
104
+ - Nd: If N=1, reset at next midnight; if N>1, reset every N days from now
105
+ - Nh: Every N hours, aligned to hour boundaries (e.g., 1:00, 2:00)
106
+ - Nm: Every N minutes, aligned to minute boundaries (e.g., 1:05, 1:10)
107
+ - Ns: Every N seconds, aligned to second boundaries
108
+
109
+ Parameters:
110
+ - duration: Duration string (e.g. "30s", "30m", "30h", "30d")
111
+ - current_time: Current datetime
112
+ - timezone_str: Timezone string (e.g. "UTC", "US/Eastern", "Asia/Kolkata")
113
+
114
+ Returns:
115
+ - Next reset time at a standardized interval in the specified timezone
116
+ """
117
+ # Set up timezone and normalize current time
118
+ current_time, tz = _setup_timezone(current_time, timezone_str)
119
+
120
+ # Parse duration
121
+ value, unit = _parse_duration(duration)
122
+ if value is None:
123
+ # Fall back to default if format is invalid
124
+ return current_time.replace(
125
+ hour=0, minute=0, second=0, microsecond=0
126
+ ) + timedelta(days=1)
127
+
128
+ # Midnight of the current day in the specified timezone
129
+ base_midnight = current_time.replace(hour=0, minute=0, second=0, microsecond=0)
130
+
131
+ # Handle different time units
132
+ if unit == "d":
133
+ return _handle_day_reset(current_time, base_midnight, value, tz)
134
+ elif unit == "h":
135
+ return _handle_hour_reset(current_time, base_midnight, value)
136
+ elif unit == "m":
137
+ return _handle_minute_reset(current_time, base_midnight, value)
138
+ elif unit == "s":
139
+ return _handle_second_reset(current_time, base_midnight, value)
140
+ elif unit == "mo":
141
+ return _handle_month_reset(current_time, base_midnight, value)
142
+ else:
143
+ # Unrecognized unit, default to next midnight
144
+ return base_midnight + timedelta(days=1)
145
+
146
+
147
+ def _setup_timezone(
148
+ current_time: datetime, timezone_str: str = "UTC"
149
+ ) -> Tuple[datetime, tzinfo]:
150
+ """Set up timezone and normalize current time to that timezone."""
151
+ try:
152
+ if timezone_str is None:
153
+ tz: tzinfo = timezone.utc
154
+ else:
155
+ tz = ZoneInfo(timezone_str)
156
+ except Exception:
157
+ # If timezone is invalid, fall back to UTC
158
+ tz = timezone.utc
159
+
160
+ # Convert current_time to the target timezone
161
+ if current_time.tzinfo is None:
162
+ # Naive datetime - assume it's UTC
163
+ utc_time = current_time.replace(tzinfo=timezone.utc)
164
+ current_time = utc_time.astimezone(tz)
165
+ else:
166
+ # Already has timezone - convert to target timezone
167
+ current_time = current_time.astimezone(tz)
168
+
169
+ return current_time, tz
170
+
171
+
172
+ def _parse_duration(duration: str) -> Tuple[Optional[int], Optional[str]]:
173
+ """Parse the duration string into value and unit."""
174
+ match = re.match(r"(\d+)([a-z]+)", duration)
175
+ if not match:
176
+ return None, None
177
+
178
+ value, unit = match.groups()
179
+ return int(value), unit
180
+
181
+
182
+ def _handle_day_reset(
183
+ current_time: datetime, base_midnight: datetime, value: int, tz: tzinfo
184
+ ) -> datetime:
185
+ """Handle day-based reset times."""
186
+ # Handle zero value - immediate expiration
187
+ if value == 0:
188
+ return current_time
189
+
190
+ if value == 1: # Daily reset at midnight
191
+ return base_midnight + timedelta(days=1)
192
+ elif value == 7: # Weekly reset on Monday at midnight
193
+ days_until_monday = (7 - current_time.weekday()) % 7
194
+ if days_until_monday == 0: # If today is Monday
195
+ days_until_monday = 7
196
+ return base_midnight + timedelta(days=days_until_monday)
197
+ elif value == 30: # Monthly reset on 1st at midnight
198
+ # Get 1st of next month at midnight
199
+ if current_time.month == 12:
200
+ next_reset = datetime(
201
+ year=current_time.year + 1,
202
+ month=1,
203
+ day=1,
204
+ hour=0,
205
+ minute=0,
206
+ second=0,
207
+ microsecond=0,
208
+ tzinfo=tz,
209
+ )
210
+ else:
211
+ next_reset = datetime(
212
+ year=current_time.year,
213
+ month=current_time.month + 1,
214
+ day=1,
215
+ hour=0,
216
+ minute=0,
217
+ second=0,
218
+ microsecond=0,
219
+ tzinfo=tz,
220
+ )
221
+ return next_reset
222
+ else: # Custom day value - next interval is value days from current
223
+ return current_time.replace(
224
+ hour=0, minute=0, second=0, microsecond=0
225
+ ) + timedelta(days=value)
226
+
227
+
228
+ def _handle_hour_reset(
229
+ current_time: datetime, base_midnight: datetime, value: int
230
+ ) -> datetime:
231
+ """Handle hour-based reset times."""
232
+ # Handle zero value - immediate expiration
233
+ if value == 0:
234
+ return current_time
235
+
236
+ current_hour = current_time.hour
237
+ current_minute = current_time.minute
238
+ current_second = current_time.second
239
+ current_microsecond = current_time.microsecond
240
+
241
+ # Calculate next hour aligned with the value
242
+ if current_minute == 0 and current_second == 0 and current_microsecond == 0:
243
+ next_hour = (
244
+ current_hour + value - (current_hour % value)
245
+ if current_hour % value != 0
246
+ else current_hour + value
247
+ )
248
+ else:
249
+ next_hour = (
250
+ current_hour + value - (current_hour % value)
251
+ if current_hour % value != 0
252
+ else current_hour + value
253
+ )
254
+
255
+ # Handle overnight case
256
+ if next_hour >= 24:
257
+ next_hour = next_hour % 24
258
+ next_day = base_midnight + timedelta(days=1)
259
+ return next_day.replace(hour=next_hour)
260
+
261
+ return current_time.replace(hour=next_hour, minute=0, second=0, microsecond=0)
262
+
263
+
264
+ def _handle_minute_reset(
265
+ current_time: datetime, base_midnight: datetime, value: int
266
+ ) -> datetime:
267
+ """Handle minute-based reset times."""
268
+ # Handle zero value - immediate expiration
269
+ if value == 0:
270
+ return current_time
271
+
272
+ current_hour = current_time.hour
273
+ current_minute = current_time.minute
274
+ current_second = current_time.second
275
+ current_microsecond = current_time.microsecond
276
+
277
+ # Calculate next minute aligned with the value
278
+ if current_second == 0 and current_microsecond == 0:
279
+ next_minute = (
280
+ current_minute + value - (current_minute % value)
281
+ if current_minute % value != 0
282
+ else current_minute + value
283
+ )
284
+ else:
285
+ next_minute = (
286
+ current_minute + value - (current_minute % value)
287
+ if current_minute % value != 0
288
+ else current_minute + value
289
+ )
290
+
291
+ # Handle hour rollover
292
+ next_hour = current_hour + (next_minute // 60)
293
+ next_minute = next_minute % 60
294
+
295
+ # Handle overnight case
296
+ if next_hour >= 24:
297
+ next_hour = next_hour % 24
298
+ next_day = base_midnight + timedelta(days=1)
299
+ return next_day.replace(
300
+ hour=next_hour, minute=next_minute, second=0, microsecond=0
301
+ )
302
+
303
+ return current_time.replace(
304
+ hour=next_hour, minute=next_minute, second=0, microsecond=0
305
+ )
306
+
307
+
308
+ def _handle_second_reset(
309
+ current_time: datetime, base_midnight: datetime, value: int
310
+ ) -> datetime:
311
+ """Handle second-based reset times."""
312
+ # Handle zero value - immediate expiration
313
+ if value == 0:
314
+ return current_time
315
+
316
+ current_hour = current_time.hour
317
+ current_minute = current_time.minute
318
+ current_second = current_time.second
319
+ current_microsecond = current_time.microsecond
320
+
321
+ # Calculate next second aligned with the value
322
+ if current_microsecond == 0:
323
+ next_second = (
324
+ current_second + value - (current_second % value)
325
+ if current_second % value != 0
326
+ else current_second + value
327
+ )
328
+ else:
329
+ next_second = (
330
+ current_second + value - (current_second % value)
331
+ if current_second % value != 0
332
+ else current_second + value
333
+ )
334
+
335
+ # Handle minute rollover
336
+ additional_minutes = next_second // 60
337
+ next_second = next_second % 60
338
+ next_minute = current_minute + additional_minutes
339
+
340
+ # Handle hour rollover
341
+ next_hour = current_hour + (next_minute // 60)
342
+ next_minute = next_minute % 60
343
+
344
+ # Handle overnight case
345
+ if next_hour >= 24:
346
+ next_hour = next_hour % 24
347
+ next_day = base_midnight + timedelta(days=1)
348
+ return next_day.replace(
349
+ hour=next_hour, minute=next_minute, second=next_second, microsecond=0
350
+ )
351
+
352
+ return current_time.replace(
353
+ hour=next_hour, minute=next_minute, second=next_second, microsecond=0
354
+ )
355
+
356
+
357
+ def _handle_month_reset(
358
+ current_time: datetime, base_midnight: datetime, value: int
359
+ ) -> datetime:
360
+ """
361
+ Handle monthly reset times. For monthly resets, we always reset at the start of the next month.
362
+
363
+ Args:
364
+ current_time: Current datetime
365
+ base_midnight: Midnight of current day
366
+ value: Number of months (currently only supports 1 month resets)
367
+
368
+ Returns:
369
+ datetime: First day of next month at midnight
370
+ """
371
+ if value != 1:
372
+ raise ValueError("Monthly resets currently only support 1 month intervals")
373
+
374
+ # Get the first day of next month
375
+ if current_time.month == 12:
376
+ next_month = 1
377
+ next_year = current_time.year + 1
378
+ else:
379
+ next_month = current_time.month + 1
380
+ next_year = current_time.year
381
+
382
+ return datetime(
383
+ year=next_year,
384
+ month=next_month,
385
+ day=1,
386
+ hour=0,
387
+ minute=0,
388
+ second=0,
389
+ microsecond=0,
390
+ tzinfo=current_time.tzinfo,
391
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/env_utils.py ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Utility helpers for reading and parsing environment variables.
3
+ """
4
+
5
+ import os
6
+
7
+
8
+ def get_env_int(env_var: str, default: int) -> int:
9
+ """Parse an environment variable as an integer, falling back to default on invalid values.
10
+
11
+ Handles empty strings, whitespace, and non-numeric values gracefully
12
+ so that misconfiguration doesn't crash the process at import time.
13
+ """
14
+ raw = os.getenv(env_var)
15
+ if raw is None:
16
+ return default
17
+ raw = raw.strip()
18
+ try:
19
+ return int(raw)
20
+ except (ValueError, TypeError):
21
+ return default
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py ADDED
The diff for this file is too large to render. See raw diff
 
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/fallback_utils.py ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from litellm._uuid import uuid
2
+ from typing import Optional
3
+
4
+ import litellm
5
+ from litellm._logging import verbose_logger
6
+ from litellm.litellm_core_utils.core_helpers import (
7
+ safe_deep_copy,
8
+ filter_internal_params,
9
+ )
10
+
11
+ from .asyncify import run_async_function
12
+
13
+
14
+ async def async_completion_with_fallbacks(**kwargs):
15
+ """
16
+ Asynchronously attempts completion with fallback models if the primary model fails.
17
+
18
+ Args:
19
+ **kwargs: Keyword arguments for completion, including:
20
+ - model (str): Primary model to use
21
+ - fallbacks (List[Union[str, dict]]): List of fallback models/configs
22
+ - Other completion parameters
23
+
24
+ Returns:
25
+ ModelResponse: The completion response from the first successful model
26
+
27
+ Raises:
28
+ Exception: If all models fail and no response is generated
29
+ """
30
+ # Extract and prepare parameters
31
+ nested_kwargs = kwargs.pop("kwargs", {})
32
+ original_model = kwargs["model"]
33
+ model = original_model
34
+ fallbacks = [original_model] + nested_kwargs.pop("fallbacks", [])
35
+ kwargs.pop("acompletion", None) # Remove to prevent keyword conflicts
36
+ litellm_call_id = str(uuid.uuid4())
37
+ base_kwargs = {**kwargs, **nested_kwargs, "litellm_call_id": litellm_call_id}
38
+
39
+ # fields to remove
40
+ base_kwargs.pop("model", None) # Remove model as it will be set per fallback
41
+ litellm_logging_obj = base_kwargs.pop("litellm_logging_obj", None)
42
+
43
+ # Try each fallback model
44
+ most_recent_exception_str: Optional[str] = None
45
+ for fallback in fallbacks:
46
+ try:
47
+ completion_kwargs = safe_deep_copy(base_kwargs)
48
+ # Handle dictionary fallback configurations
49
+ if isinstance(fallback, dict):
50
+ model = fallback.pop("model", original_model)
51
+ completion_kwargs.update(fallback)
52
+ else:
53
+ model = fallback
54
+
55
+ # Filter out internal parameters that shouldn't be sent to provider APIs
56
+ completion_kwargs = filter_internal_params(completion_kwargs)
57
+
58
+ response = await litellm.acompletion(
59
+ **completion_kwargs,
60
+ model=model,
61
+ litellm_logging_obj=litellm_logging_obj,
62
+ )
63
+
64
+ if response is not None:
65
+ return response
66
+
67
+ except Exception as e:
68
+ verbose_logger.exception(
69
+ f"Fallback attempt failed for model {model}: {str(e)}"
70
+ )
71
+ most_recent_exception_str = str(e)
72
+ continue
73
+
74
+ raise Exception(
75
+ f"{most_recent_exception_str}. All fallback attempts failed. Enable verbose logging with `litellm.set_verbose=True` for details."
76
+ )
77
+
78
+
79
+ def completion_with_fallbacks(**kwargs):
80
+ return run_async_function(async_function=async_completion_with_fallbacks, **kwargs)
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_blog_posts.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Pulls the latest LiteLLM blog posts from the docs RSS feed.
3
+
4
+ Falls back to the bundled local backup on any failure.
5
+ RSS URL is configured via litellm.blog_posts_url (or LITELLM_BLOG_POSTS_URL env var).
6
+
7
+ Disable remote fetching entirely:
8
+ export LITELLM_LOCAL_BLOG_POSTS=True
9
+ """
10
+
11
+ import json
12
+ import os
13
+ import time
14
+ import xml.etree.ElementTree as ET
15
+ from email.utils import parsedate_to_datetime
16
+ from importlib.resources import files
17
+ from typing import Dict, List, Optional
18
+
19
+ import httpx
20
+ from pydantic import BaseModel
21
+
22
+ from litellm import verbose_logger
23
+
24
+ BLOG_POSTS_TTL_SECONDS: int = 3600 # 1 hour
25
+
26
+
27
+ class BlogPost(BaseModel):
28
+ title: str
29
+ description: str
30
+ date: str
31
+ url: str
32
+
33
+
34
+ class BlogPostsResponse(BaseModel):
35
+ posts: List[BlogPost]
36
+
37
+
38
+ class GetBlogPosts:
39
+ """
40
+ Fetches, validates, and caches LiteLLM blog posts.
41
+
42
+ - Fetches RSS feed from docs site with a 5-second timeout
43
+ - Parses the XML and extracts the latest blog post
44
+ - Caches the result in-process for BLOG_POSTS_TTL_SECONDS (1 hour)
45
+ - Falls back to the bundled local backup on any failure
46
+ """
47
+
48
+ _cached_posts: Optional[List[Dict[str, str]]] = None
49
+ _last_fetch_time: float = 0.0
50
+
51
+ @staticmethod
52
+ def load_local_blog_posts() -> List[Dict[str, str]]:
53
+ """Load the bundled local backup blog posts."""
54
+ content = json.loads(
55
+ files("litellm").joinpath("blog_posts.json").read_text(encoding="utf-8")
56
+ )
57
+ return content.get("posts", [])
58
+
59
+ @staticmethod
60
+ def fetch_rss_feed(url: str, timeout: int = 5) -> str:
61
+ """
62
+ Fetch RSS XML from a remote URL.
63
+
64
+ Returns the raw XML text. Raises on network errors.
65
+ """
66
+ response = httpx.get(url, timeout=timeout)
67
+ response.raise_for_status()
68
+ return response.text
69
+
70
+ @staticmethod
71
+ def parse_rss_to_posts(xml_text: str, max_posts: int = 1) -> List[Dict[str, str]]:
72
+ """
73
+ Parse RSS XML and return a list of blog post dicts.
74
+
75
+ Extracts title, description, date (YYYY-MM-DD), and url from each <item>.
76
+ """
77
+ root = ET.fromstring(xml_text)
78
+ channel = root.find("channel")
79
+ if channel is None:
80
+ raise ValueError("RSS feed missing <channel> element")
81
+
82
+ posts: List[Dict[str, str]] = []
83
+ for item in channel.findall("item"):
84
+ if len(posts) >= max_posts:
85
+ break
86
+
87
+ title_el = item.find("title")
88
+ link_el = item.find("link")
89
+ desc_el = item.find("description")
90
+ pub_date_el = item.find("pubDate")
91
+
92
+ if title_el is None or link_el is None:
93
+ continue
94
+
95
+ # Parse RFC 2822 date to YYYY-MM-DD
96
+ date_str = ""
97
+ if pub_date_el is not None and pub_date_el.text:
98
+ try:
99
+ dt = parsedate_to_datetime(pub_date_el.text)
100
+ date_str = dt.strftime("%Y-%m-%d")
101
+ except Exception:
102
+ date_str = pub_date_el.text
103
+
104
+ posts.append(
105
+ {
106
+ "title": title_el.text or "",
107
+ "description": desc_el.text or "" if desc_el is not None else "",
108
+ "date": date_str,
109
+ "url": link_el.text or "",
110
+ }
111
+ )
112
+
113
+ return posts
114
+
115
+ @staticmethod
116
+ def validate_blog_posts(posts: List[Dict[str, str]]) -> bool:
117
+ """Return True if posts is a non-empty list."""
118
+ if not isinstance(posts, list) or len(posts) == 0:
119
+ verbose_logger.warning(
120
+ "LiteLLM: Parsed RSS feed has no valid posts. "
121
+ "Falling back to local backup.",
122
+ )
123
+ return False
124
+ return True
125
+
126
+ @classmethod
127
+ def get_blog_posts(cls, url: str) -> List[Dict[str, str]]:
128
+ """
129
+ Return the blog posts list.
130
+
131
+ Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS.
132
+ Fetches from ``url`` otherwise, falling back to local backup on failure.
133
+ """
134
+ if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
135
+ return cls.load_local_blog_posts()
136
+
137
+ now = time.time()
138
+ cached = cls._cached_posts
139
+ if cached is not None and (now - cls._last_fetch_time) < BLOG_POSTS_TTL_SECONDS:
140
+ return cached
141
+
142
+ try:
143
+ xml_text = cls.fetch_rss_feed(url)
144
+ posts = cls.parse_rss_to_posts(xml_text)
145
+ except Exception as e:
146
+ verbose_logger.warning(
147
+ "LiteLLM: Failed to fetch blog posts from %s: %s. "
148
+ "Falling back to local backup.",
149
+ url,
150
+ str(e),
151
+ )
152
+ return cls.load_local_blog_posts()
153
+
154
+ if not cls.validate_blog_posts(posts):
155
+ return cls.load_local_blog_posts()
156
+
157
+ cls._cached_posts = posts
158
+ cls._last_fetch_time = now
159
+ return posts
160
+
161
+
162
+ def get_blog_posts(url: str) -> List[Dict[str, str]]:
163
+ """Public entry point — returns the blog posts list."""
164
+ return GetBlogPosts.get_blog_posts(url=url)
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_litellm_params.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional
2
+
3
+ # Pre-define optional kwargs keys as frozenset for O(1) lookups
4
+ # These are extracted from kwargs only if present, avoiding unnecessary .get() calls
5
+ _OPTIONAL_KWARGS_KEYS = frozenset(
6
+ {
7
+ "azure_ad_token",
8
+ "tenant_id",
9
+ "client_id",
10
+ "client_secret",
11
+ "azure_username",
12
+ "azure_password",
13
+ "azure_scope",
14
+ "timeout",
15
+ "bucket_name",
16
+ "vertex_credentials",
17
+ "vertex_project",
18
+ "vertex_location",
19
+ "vertex_ai_project",
20
+ "vertex_ai_location",
21
+ "vertex_ai_credentials",
22
+ "aws_region_name",
23
+ "aws_access_key_id",
24
+ "aws_secret_access_key",
25
+ "aws_session_token",
26
+ "aws_session_name",
27
+ "aws_profile_name",
28
+ "aws_role_name",
29
+ "aws_web_identity_token",
30
+ "aws_sts_endpoint",
31
+ "aws_external_id",
32
+ "aws_bedrock_runtime_endpoint",
33
+ "tpm",
34
+ "rpm",
35
+ }
36
+ )
37
+
38
+
39
+ def _get_base_model_from_litellm_call_metadata(
40
+ metadata: Optional[dict],
41
+ ) -> Optional[str]:
42
+ if metadata is None:
43
+ return None
44
+ model_info = metadata.get("model_info")
45
+ if model_info:
46
+ return model_info.get("base_model")
47
+ return None
48
+
49
+
50
+ def get_litellm_params(
51
+ api_key: Optional[str] = None,
52
+ force_timeout=600,
53
+ azure=False,
54
+ logger_fn=None,
55
+ verbose=False,
56
+ hugging_face=False,
57
+ replicate=False,
58
+ together_ai=False,
59
+ custom_llm_provider: Optional[str] = None,
60
+ api_base: Optional[str] = None,
61
+ litellm_call_id=None,
62
+ model_alias_map=None,
63
+ completion_call_id=None,
64
+ metadata: Optional[dict] = None,
65
+ model_info=None,
66
+ proxy_server_request=None,
67
+ acompletion=None,
68
+ aembedding=None,
69
+ preset_cache_key=None,
70
+ no_log=None,
71
+ input_cost_per_second=None,
72
+ input_cost_per_token=None,
73
+ output_cost_per_token=None,
74
+ output_cost_per_second=None,
75
+ cost_per_query=None,
76
+ cooldown_time=None,
77
+ text_completion=None,
78
+ azure_ad_token_provider=None,
79
+ user_continue_message=None,
80
+ base_model: Optional[str] = None,
81
+ litellm_trace_id: Optional[str] = None,
82
+ litellm_session_id: Optional[str] = None,
83
+ hf_model_name: Optional[str] = None,
84
+ custom_prompt_dict: Optional[dict] = None,
85
+ litellm_metadata: Optional[dict] = None,
86
+ disable_add_transform_inline_image_block: Optional[bool] = None,
87
+ drop_params: Optional[bool] = None,
88
+ prompt_id: Optional[str] = None,
89
+ prompt_variables: Optional[dict] = None,
90
+ async_call: Optional[bool] = None,
91
+ ssl_verify: Optional[bool] = None,
92
+ merge_reasoning_content_in_choices: Optional[bool] = None,
93
+ use_litellm_proxy: Optional[bool] = None,
94
+ api_version: Optional[str] = None,
95
+ max_retries: Optional[int] = None,
96
+ litellm_request_debug: Optional[bool] = None,
97
+ **kwargs,
98
+ ) -> dict:
99
+ # Derive litellm_session_id / litellm_trace_id from metadata when not provided (call chaining)
100
+ _meta = metadata or {}
101
+ if litellm_session_id is None:
102
+ litellm_session_id = _meta.get("session_id") or _meta.get("trace_id")
103
+ if litellm_trace_id is None:
104
+ litellm_trace_id = _meta.get("trace_id") or _meta.get("session_id")
105
+
106
+ # Build base dict with explicit parameters (always included)
107
+ litellm_params = {
108
+ "acompletion": acompletion,
109
+ "api_key": api_key,
110
+ "force_timeout": force_timeout,
111
+ "logger_fn": logger_fn,
112
+ "verbose": verbose,
113
+ "custom_llm_provider": custom_llm_provider,
114
+ "api_base": api_base,
115
+ "litellm_call_id": litellm_call_id,
116
+ "model_alias_map": model_alias_map,
117
+ "completion_call_id": completion_call_id,
118
+ "aembedding": aembedding,
119
+ "metadata": metadata,
120
+ "model_info": model_info,
121
+ "proxy_server_request": proxy_server_request,
122
+ "preset_cache_key": preset_cache_key,
123
+ "no-log": no_log or kwargs.get("no-log"),
124
+ "stream_response": {}, # litellm_call_id: ModelResponse Dict
125
+ "input_cost_per_token": input_cost_per_token,
126
+ "input_cost_per_second": input_cost_per_second,
127
+ "output_cost_per_token": output_cost_per_token,
128
+ "output_cost_per_second": output_cost_per_second,
129
+ "cost_per_query": cost_per_query,
130
+ "cooldown_time": cooldown_time,
131
+ "text_completion": text_completion,
132
+ "azure_ad_token_provider": azure_ad_token_provider,
133
+ "user_continue_message": user_continue_message,
134
+ "base_model": base_model
135
+ or (
136
+ _get_base_model_from_litellm_call_metadata(metadata=metadata)
137
+ if metadata
138
+ else None
139
+ ),
140
+ "litellm_trace_id": litellm_trace_id,
141
+ "litellm_session_id": litellm_session_id,
142
+ "hf_model_name": hf_model_name,
143
+ "custom_prompt_dict": custom_prompt_dict,
144
+ "litellm_metadata": litellm_metadata,
145
+ "disable_add_transform_inline_image_block": disable_add_transform_inline_image_block,
146
+ "drop_params": drop_params,
147
+ "prompt_id": prompt_id,
148
+ "prompt_variables": prompt_variables,
149
+ "async_call": async_call,
150
+ "ssl_verify": ssl_verify,
151
+ "merge_reasoning_content_in_choices": merge_reasoning_content_in_choices,
152
+ "api_version": api_version,
153
+ "max_retries": max_retries,
154
+ "use_litellm_proxy": use_litellm_proxy,
155
+ "litellm_request_debug": litellm_request_debug,
156
+ }
157
+
158
+ # Sparse extraction: only add kwargs keys that are actually present
159
+ if kwargs:
160
+ for key in _OPTIONAL_KWARGS_KEYS:
161
+ if key in kwargs:
162
+ litellm_params[key] = kwargs[key]
163
+
164
+ return litellm_params
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_llm_provider_logic.py ADDED
@@ -0,0 +1,958 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional, Tuple
2
+
3
+ import litellm
4
+ from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
5
+ from litellm.llms.openai_like.json_loader import JSONProviderRegistry
6
+ from litellm.secret_managers.main import get_secret, get_secret_str
7
+
8
+ from ..types.router import LiteLLM_Params
9
+
10
+
11
+ def _is_non_openai_azure_model(model: str) -> bool:
12
+ try:
13
+ model_name = model.split("/", 1)[1]
14
+ if (
15
+ model_name in litellm.cohere_chat_models
16
+ or f"mistral/{model_name}" in litellm.mistral_chat_models
17
+ ):
18
+ return True
19
+ except Exception:
20
+ return False
21
+ return False
22
+
23
+
24
+ def _is_azure_claude_model(model: str) -> bool:
25
+ """
26
+ Check if a model name contains 'claude' (case-insensitive).
27
+ Used to detect Claude models that need Anthropic-specific handling.
28
+ """
29
+ try:
30
+ model_lower = model.lower()
31
+ return "claude" in model_lower or model_lower.startswith("claude")
32
+ except Exception:
33
+ return False
34
+
35
+
36
+ def handle_cohere_chat_model_custom_llm_provider(
37
+ model: str, custom_llm_provider: Optional[str] = None
38
+ ) -> Tuple[str, Optional[str]]:
39
+ """
40
+ if user sets model = "cohere/command-r" -> use custom_llm_provider = "cohere_chat"
41
+
42
+ Args:
43
+ model:
44
+ custom_llm_provider:
45
+
46
+ Returns:
47
+ model, custom_llm_provider
48
+ """
49
+
50
+ if custom_llm_provider:
51
+ if custom_llm_provider == "cohere" and model in litellm.cohere_chat_models:
52
+ return model, "cohere_chat"
53
+
54
+ if model and "/" in model:
55
+ _custom_llm_provider, _model = model.split("/", 1)
56
+ if (
57
+ _custom_llm_provider
58
+ and _custom_llm_provider == "cohere"
59
+ and _model in litellm.cohere_chat_models
60
+ ):
61
+ return _model, "cohere_chat"
62
+
63
+ return model, custom_llm_provider
64
+
65
+
66
+ def handle_anthropic_text_model_custom_llm_provider(
67
+ model: str, custom_llm_provider: Optional[str] = None
68
+ ) -> Tuple[str, Optional[str]]:
69
+ """
70
+ if user sets model = "anthropic/claude-2" -> use custom_llm_provider = "anthropic_text"
71
+
72
+ Args:
73
+ model:
74
+ custom_llm_provider:
75
+
76
+ Returns:
77
+ model, custom_llm_provider
78
+ """
79
+
80
+ if custom_llm_provider:
81
+ if (
82
+ custom_llm_provider == "anthropic"
83
+ and litellm.AnthropicTextConfig._is_anthropic_text_model(model)
84
+ ):
85
+ return model, "anthropic_text"
86
+
87
+ if model and "/" in model:
88
+ _custom_llm_provider, _model = model.split("/", 1)
89
+ if (
90
+ _custom_llm_provider
91
+ and _custom_llm_provider == "anthropic"
92
+ and litellm.AnthropicTextConfig._is_anthropic_text_model(_model)
93
+ ):
94
+ return _model, "anthropic_text"
95
+
96
+ return model, custom_llm_provider
97
+
98
+
99
+ def get_llm_provider( # noqa: PLR0915
100
+ model: str,
101
+ custom_llm_provider: Optional[str] = None,
102
+ api_base: Optional[str] = None,
103
+ api_key: Optional[str] = None,
104
+ litellm_params: Optional[LiteLLM_Params] = None,
105
+ ) -> Tuple[str, str, Optional[str], Optional[str]]:
106
+ """
107
+ Returns the provider for a given model name - e.g. 'azure/chatgpt-v-2' -> 'azure'
108
+
109
+ For router -> Can also give the whole litellm param dict -> this function will extract the relevant details
110
+
111
+ Raises Error - if unable to map model to a provider
112
+
113
+ Return model, custom_llm_provider, dynamic_api_key, api_base
114
+ """
115
+ try:
116
+ # Early validation - model is required
117
+ if model is None:
118
+ raise ValueError(
119
+ "model parameter is required but was None. Please provide a valid model name."
120
+ )
121
+
122
+ if litellm.LiteLLMProxyChatConfig._should_use_litellm_proxy_by_default(
123
+ litellm_params=litellm_params
124
+ ):
125
+ return litellm.LiteLLMProxyChatConfig.litellm_proxy_get_custom_llm_provider_info(
126
+ model=model, api_base=api_base, api_key=api_key
127
+ )
128
+
129
+ ## IF LITELLM PARAMS GIVEN ##
130
+ if litellm_params:
131
+ assert (
132
+ custom_llm_provider is None and api_base is None and api_key is None
133
+ ), "Either pass in litellm_params or the custom_llm_provider/api_base/api_key. Otherwise, these values will be overriden."
134
+ custom_llm_provider = litellm_params.custom_llm_provider
135
+ api_base = litellm_params.api_base
136
+ api_key = litellm_params.api_key
137
+
138
+ dynamic_api_key = None
139
+ # check if llm provider provided
140
+ # AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere
141
+ # If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus
142
+ if model.split("/", 1)[0] == "azure":
143
+ if _is_non_openai_azure_model(model):
144
+ custom_llm_provider = "openai"
145
+ return model, custom_llm_provider, dynamic_api_key, api_base
146
+
147
+ ### Handle cases when custom_llm_provider is set to cohere/command-r-plus but it should use cohere_chat route
148
+ model, custom_llm_provider = handle_cohere_chat_model_custom_llm_provider(
149
+ model, custom_llm_provider
150
+ )
151
+
152
+ model, custom_llm_provider = handle_anthropic_text_model_custom_llm_provider(
153
+ model, custom_llm_provider
154
+ )
155
+
156
+ if custom_llm_provider and (
157
+ model.split("/")[0] != custom_llm_provider
158
+ ): # handle scenario where model="azure/*" and custom_llm_provider="azure"
159
+ model = custom_llm_provider + "/" + model
160
+
161
+ # Native OpenRouter models have IDs like "openrouter/free" where the
162
+ # "openrouter/" prefix is part of the actual model name on the API.
163
+ # When called from a bridge (e.g. anthropic_messages adapter),
164
+ # custom_llm_provider is already resolved, so return early to prevent
165
+ # the provider-list stripping below from removing the prefix.
166
+ if custom_llm_provider == "openrouter" and model.startswith("openrouter/"):
167
+ return model, custom_llm_provider, dynamic_api_key, api_base
168
+
169
+ if api_key and api_key.startswith("os.environ/"):
170
+ dynamic_api_key = get_secret_str(api_key)
171
+
172
+ # Check JSON-configured providers FIRST (before enum-based provider_list)
173
+ provider_prefix = model.split("/", 1)[0]
174
+ if len(model.split("/")) > 1 and JSONProviderRegistry.exists(provider_prefix):
175
+ return _get_openai_compatible_provider_info(
176
+ model=model,
177
+ api_base=api_base,
178
+ api_key=api_key,
179
+ dynamic_api_key=dynamic_api_key,
180
+ )
181
+
182
+ # check if llm provider part of model name
183
+
184
+ if (
185
+ model.split("/", 1)[0] in litellm.provider_list
186
+ and model.split("/", 1)[0] not in litellm.model_list_set
187
+ and len(model.split("/"))
188
+ > 1 # handle edge case where user passes in `litellm --model mistral` https://github.com/BerriAI/litellm/issues/1351
189
+ ):
190
+ return _get_openai_compatible_provider_info(
191
+ model=model,
192
+ api_base=api_base,
193
+ api_key=api_key,
194
+ dynamic_api_key=dynamic_api_key,
195
+ )
196
+ elif model.split("/", 1)[0] in litellm.provider_list:
197
+ custom_llm_provider = model.split("/", 1)[0]
198
+ model = model.split("/", 1)[1]
199
+ if api_base is not None and not isinstance(api_base, str):
200
+ raise Exception(
201
+ "api base needs to be a string. api_base={}".format(api_base)
202
+ )
203
+ if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
204
+ raise Exception(
205
+ "dynamic_api_key needs to be a string. dynamic_api_key={}".format(
206
+ dynamic_api_key
207
+ )
208
+ )
209
+ return model, custom_llm_provider, dynamic_api_key, api_base
210
+ # check if api base is a known openai compatible endpoint
211
+ if api_base:
212
+ for endpoint in litellm.openai_compatible_endpoints:
213
+ if endpoint in api_base:
214
+ if endpoint == "api.perplexity.ai":
215
+ custom_llm_provider = "perplexity"
216
+ dynamic_api_key = get_secret_str("PERPLEXITYAI_API_KEY")
217
+ elif endpoint == "api.endpoints.anyscale.com/v1":
218
+ custom_llm_provider = "anyscale"
219
+ dynamic_api_key = get_secret_str("ANYSCALE_API_KEY")
220
+ elif endpoint == "api.deepinfra.com/v1/openai":
221
+ custom_llm_provider = "deepinfra"
222
+ dynamic_api_key = get_secret_str("DEEPINFRA_API_KEY")
223
+ elif endpoint == "api.mistral.ai/v1":
224
+ custom_llm_provider = "mistral"
225
+ dynamic_api_key = get_secret_str("MISTRAL_API_KEY")
226
+ elif endpoint == "api.groq.com/openai/v1":
227
+ custom_llm_provider = "groq"
228
+ dynamic_api_key = get_secret_str("GROQ_API_KEY")
229
+ elif endpoint == "https://integrate.api.nvidia.com/v1":
230
+ custom_llm_provider = "nvidia_nim"
231
+ dynamic_api_key = get_secret_str("NVIDIA_NIM_API_KEY")
232
+ elif endpoint == "https://api.cerebras.ai/v1":
233
+ custom_llm_provider = "cerebras"
234
+ dynamic_api_key = get_secret_str("CEREBRAS_API_KEY")
235
+ elif endpoint == "https://inference.baseten.co/v1":
236
+ custom_llm_provider = "baseten"
237
+ dynamic_api_key = get_secret_str("BASETEN_API_KEY")
238
+ elif endpoint == "https://api.sambanova.ai/v1":
239
+ custom_llm_provider = "sambanova"
240
+ dynamic_api_key = get_secret_str("SAMBANOVA_API_KEY")
241
+ elif endpoint == "https://api.ai21.com/studio/v1":
242
+ custom_llm_provider = "ai21_chat"
243
+ dynamic_api_key = get_secret_str("AI21_API_KEY")
244
+ elif endpoint == "codestral.mistral.ai/v1/chat/completions":
245
+ custom_llm_provider = "codestral"
246
+ dynamic_api_key = get_secret_str("CODESTRAL_API_KEY")
247
+ elif endpoint == "codestral.mistral.ai/v1/fim/completions":
248
+ custom_llm_provider = "text-completion-codestral"
249
+ dynamic_api_key = get_secret_str("CODESTRAL_API_KEY")
250
+ elif endpoint == "app.empower.dev/api/v1":
251
+ custom_llm_provider = "empower"
252
+ dynamic_api_key = get_secret_str("EMPOWER_API_KEY")
253
+ elif endpoint == "api.deepseek.com/v1":
254
+ custom_llm_provider = "deepseek"
255
+ dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY")
256
+ elif endpoint == "ollama.com":
257
+ custom_llm_provider = "ollama"
258
+ dynamic_api_key = get_secret_str("OLLAMA_API_KEY")
259
+ elif endpoint == "https://api.friendli.ai/serverless/v1":
260
+ custom_llm_provider = "friendliai"
261
+ dynamic_api_key = get_secret_str(
262
+ "FRIENDLIAI_API_KEY"
263
+ ) or get_secret("FRIENDLI_TOKEN")
264
+ elif endpoint == "api.galadriel.com/v1":
265
+ custom_llm_provider = "galadriel"
266
+ dynamic_api_key = get_secret_str("GALADRIEL_API_KEY")
267
+ elif endpoint == "https://api.llama.com/compat/v1":
268
+ custom_llm_provider = "meta_llama"
269
+ dynamic_api_key = api_key or get_secret_str("LLAMA_API_KEY")
270
+ elif endpoint == "https://api.featherless.ai/v1":
271
+ custom_llm_provider = "featherless_ai"
272
+ dynamic_api_key = get_secret_str("FEATHERLESS_AI_API_KEY")
273
+ elif endpoint == litellm.NscaleConfig.API_BASE_URL:
274
+ custom_llm_provider = "nscale"
275
+ dynamic_api_key = litellm.NscaleConfig.get_api_key()
276
+ elif endpoint == "dashscope-intl.aliyuncs.com/compatible-mode/v1":
277
+ custom_llm_provider = "dashscope"
278
+ dynamic_api_key = get_secret_str("DASHSCOPE_API_KEY")
279
+ elif endpoint == "api.moonshot.ai/v1":
280
+ custom_llm_provider = "moonshot"
281
+ dynamic_api_key = get_secret_str("MOONSHOT_API_KEY")
282
+ elif (
283
+ endpoint == "api.minimax.io/anthropic"
284
+ or endpoint == "api.minimaxi.com/anthropic"
285
+ ):
286
+ custom_llm_provider = "minimax"
287
+ dynamic_api_key = get_secret_str("MINIMAX_API_KEY")
288
+ elif (
289
+ endpoint == "api.minimax.io/v1"
290
+ or endpoint == "api.minimaxi.com/v1"
291
+ ):
292
+ custom_llm_provider = "minimax"
293
+ dynamic_api_key = get_secret_str("MINIMAX_API_KEY")
294
+ elif endpoint == "platform.publicai.co/v1":
295
+ custom_llm_provider = "publicai"
296
+ dynamic_api_key = get_secret_str("PUBLICAI_API_KEY")
297
+ elif endpoint == "https://api.synthetic.new/openai/v1":
298
+ custom_llm_provider = "synthetic"
299
+ dynamic_api_key = get_secret_str("SYNTHETIC_API_KEY")
300
+ elif endpoint == "https://api.stima.tech/v1":
301
+ custom_llm_provider = "apertis"
302
+ dynamic_api_key = get_secret_str("STIMA_API_KEY")
303
+ elif endpoint == "https://nano-gpt.com/api/v1":
304
+ custom_llm_provider = "nano-gpt"
305
+ dynamic_api_key = get_secret_str("NANOGPT_API_KEY")
306
+ elif endpoint == "https://api.poe.com/v1":
307
+ custom_llm_provider = "poe"
308
+ dynamic_api_key = get_secret_str("POE_API_KEY")
309
+ elif endpoint == "https://llm.chutes.ai/v1/":
310
+ custom_llm_provider = "chutes"
311
+ dynamic_api_key = get_secret_str("CHUTES_API_KEY")
312
+ elif endpoint == "https://api.v0.dev/v1":
313
+ custom_llm_provider = "v0"
314
+ dynamic_api_key = get_secret_str("V0_API_KEY")
315
+ elif endpoint == "https://api.lambda.ai/v1":
316
+ custom_llm_provider = "lambda_ai"
317
+ dynamic_api_key = get_secret_str("LAMBDA_API_KEY")
318
+ elif endpoint == "https://api.hyperbolic.xyz/v1":
319
+ custom_llm_provider = "hyperbolic"
320
+ dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY")
321
+ elif endpoint == "https://ai-gateway.vercel.sh/v1":
322
+ custom_llm_provider = "vercel_ai_gateway"
323
+ dynamic_api_key = get_secret_str("VERCEL_AI_GATEWAY_API_KEY")
324
+ elif endpoint == "https://api.inference.wandb.ai/v1":
325
+ custom_llm_provider = "wandb"
326
+ dynamic_api_key = get_secret_str("WANDB_API_KEY")
327
+
328
+ if api_base is not None and not isinstance(api_base, str):
329
+ raise Exception(
330
+ "api base needs to be a string. api_base={}".format(
331
+ api_base
332
+ )
333
+ )
334
+ if dynamic_api_key is not None and not isinstance(
335
+ dynamic_api_key, str
336
+ ):
337
+ raise Exception(
338
+ "dynamic_api_key needs to be a string. dynamic_api_key={}".format(
339
+ dynamic_api_key
340
+ )
341
+ )
342
+ return model, custom_llm_provider, dynamic_api_key, api_base # type: ignore
343
+
344
+ # check if model in known model provider list -> for huggingface models, raise exception as they don't have a fixed provider (can be togetherai, anyscale, baseten, runpod, et.)
345
+ ## openai - chatcompletion + text completion
346
+ if (
347
+ model in litellm.open_ai_chat_completion_models
348
+ or "ft:gpt-3.5-turbo" in model
349
+ or "ft:gpt-4" in model # catches ft:gpt-4-0613, ft:gpt-4o
350
+ or model in litellm.openai_image_generation_models
351
+ or model in litellm.openai_video_generation_models
352
+ ):
353
+ custom_llm_provider = "openai"
354
+ elif model in litellm.open_ai_text_completion_models:
355
+ custom_llm_provider = "text-completion-openai"
356
+ ## anthropic
357
+ elif model in litellm.anthropic_models:
358
+ if litellm.AnthropicTextConfig._is_anthropic_text_model(model):
359
+ custom_llm_provider = "anthropic_text"
360
+ else:
361
+ custom_llm_provider = "anthropic"
362
+ ## cohere
363
+ elif model in litellm.cohere_models or model in litellm.cohere_embedding_models:
364
+ custom_llm_provider = "cohere"
365
+ ## cohere chat models
366
+ elif model in litellm.cohere_chat_models:
367
+ custom_llm_provider = "cohere_chat"
368
+ ## replicate
369
+ elif model in litellm.replicate_models or (
370
+ ":" in model and len(model) > REPLICATE_MODEL_NAME_WITH_ID_LENGTH
371
+ ):
372
+ model_parts = model.split(":")
373
+ if (
374
+ len(model_parts) > 1
375
+ and len(model_parts[1]) == REPLICATE_MODEL_NAME_WITH_ID_LENGTH
376
+ ): ## checks if model name has a 64 digit code - e.g. "meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3"
377
+ custom_llm_provider = "replicate"
378
+ elif model in litellm.replicate_models:
379
+ custom_llm_provider = "replicate"
380
+ ## openrouter
381
+ elif model in litellm.openrouter_models:
382
+ custom_llm_provider = "openrouter"
383
+ ## maritalk
384
+ elif model in litellm.maritalk_models:
385
+ custom_llm_provider = "maritalk"
386
+ ## vertex - text + chat + language (gemini) models
387
+ elif (
388
+ model in litellm.vertex_chat_models
389
+ or model in litellm.vertex_code_chat_models
390
+ or model in litellm.vertex_text_models
391
+ or model in litellm.vertex_code_text_models
392
+ or model in litellm.vertex_language_models
393
+ or model in litellm.vertex_embedding_models
394
+ or model in litellm.vertex_vision_models
395
+ or model in litellm.vertex_ai_image_models
396
+ or model in litellm.vertex_ai_video_models
397
+ ):
398
+ custom_llm_provider = "vertex_ai"
399
+ ## ai21
400
+ elif model in litellm.ai21_chat_models or model in litellm.ai21_models:
401
+ custom_llm_provider = "ai21_chat"
402
+ api_base = (
403
+ api_base
404
+ or get_secret("AI21_API_BASE")
405
+ or "https://api.ai21.com/studio/v1"
406
+ ) # type: ignore
407
+ dynamic_api_key = api_key or get_secret("AI21_API_KEY")
408
+ ## aleph_alpha
409
+ elif model in litellm.aleph_alpha_models:
410
+ custom_llm_provider = "aleph_alpha"
411
+ ## baseten
412
+ elif model in litellm.baseten_models:
413
+ custom_llm_provider = "baseten"
414
+ ## nlp_cloud
415
+ elif model in litellm.nlp_cloud_models:
416
+ custom_llm_provider = "nlp_cloud"
417
+ ## petals
418
+ elif model in litellm.petals_models:
419
+ custom_llm_provider = "petals"
420
+ ## bedrock
421
+ elif (
422
+ model in litellm.bedrock_models
423
+ or model in litellm.bedrock_embedding_models
424
+ or model in litellm.bedrock_converse_models
425
+ ):
426
+ custom_llm_provider = "bedrock"
427
+ elif model in litellm.watsonx_models:
428
+ custom_llm_provider = "watsonx"
429
+ # openai embeddings
430
+ elif model in litellm.open_ai_embedding_models:
431
+ custom_llm_provider = "openai"
432
+ elif model in litellm.empower_models:
433
+ custom_llm_provider = "empower"
434
+ elif model in litellm.gradient_ai_models:
435
+ custom_llm_provider = "gradient_ai"
436
+ elif model == "*":
437
+ custom_llm_provider = "openai"
438
+ # bytez models
439
+ elif model.startswith("bytez/"):
440
+ custom_llm_provider = "bytez"
441
+ elif model.startswith("lemonade/"):
442
+ custom_llm_provider = "lemonade"
443
+ elif model.startswith("heroku/"):
444
+ custom_llm_provider = "heroku"
445
+ # cometapi models
446
+ elif model.startswith("cometapi/"):
447
+ custom_llm_provider = "cometapi"
448
+ elif model.startswith("oci/"):
449
+ custom_llm_provider = "oci"
450
+ elif model.startswith("compactifai/"):
451
+ custom_llm_provider = "compactifai"
452
+ elif model.startswith("ovhcloud/"):
453
+ custom_llm_provider = "ovhcloud"
454
+ elif model.startswith("lemonade/"):
455
+ custom_llm_provider = "lemonade"
456
+ elif model.startswith("clarifai/"):
457
+ custom_llm_provider = "clarifai"
458
+ elif model.startswith("amazon_nova"):
459
+ custom_llm_provider = "amazon_nova"
460
+ elif model.startswith("sap/"):
461
+ custom_llm_provider = "sap"
462
+ if not custom_llm_provider:
463
+ if litellm.suppress_debug_info is False:
464
+ print() # noqa
465
+ print( # noqa
466
+ "\033[1;31mProvider List: https://docs.litellm.ai/docs/providers\033[0m" # noqa
467
+ ) # noqa
468
+ print() # noqa
469
+ error_str = f"LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model={model}\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers"
470
+ # maps to openai.NotFoundError, this is raised when openai does not recognize the llm
471
+ raise litellm.exceptions.BadRequestError( # type: ignore
472
+ message=error_str,
473
+ model=model,
474
+ response=None,
475
+ llm_provider="",
476
+ )
477
+ if api_base is not None and not isinstance(api_base, str):
478
+ raise Exception(
479
+ "api base needs to be a string. api_base={}".format(api_base)
480
+ )
481
+ if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
482
+ raise Exception(
483
+ "dynamic_api_key needs to be a string. dynamic_api_key={}".format(
484
+ dynamic_api_key
485
+ )
486
+ )
487
+ return model, custom_llm_provider, dynamic_api_key, api_base
488
+ except Exception as e:
489
+ if isinstance(e, litellm.exceptions.BadRequestError):
490
+ raise e
491
+ else:
492
+ error_str = (
493
+ f"GetLLMProvider Exception - {str(e)}\n\noriginal model: {model}"
494
+ )
495
+ raise litellm.exceptions.BadRequestError( # type: ignore
496
+ message=f"GetLLMProvider Exception - {str(e)}\n\noriginal model: {model}",
497
+ model=model,
498
+ response=None,
499
+ llm_provider="",
500
+ )
501
+
502
+
503
+ def _get_openai_compatible_provider_info( # noqa: PLR0915
504
+ model: str,
505
+ api_base: Optional[str],
506
+ api_key: Optional[str],
507
+ dynamic_api_key: Optional[str],
508
+ ) -> Tuple[str, str, Optional[str], Optional[str]]:
509
+ """
510
+ Returns:
511
+ Tuple[str, str, Optional[str], Optional[str]]:
512
+ model: str
513
+ custom_llm_provider: str
514
+ dynamic_api_key: Optional[str]
515
+ api_base: Optional[str]
516
+ """
517
+
518
+ custom_llm_provider = model.split("/", 1)[0]
519
+ model = model.split("/", 1)[1]
520
+
521
+ # Check JSON providers FIRST (before hardcoded ones)
522
+ from litellm.llms.openai_like.dynamic_config import create_config_class
523
+ from litellm.llms.openai_like.json_loader import JSONProviderRegistry
524
+
525
+ if JSONProviderRegistry.exists(custom_llm_provider):
526
+ provider_config = JSONProviderRegistry.get(custom_llm_provider)
527
+ if provider_config is None:
528
+ raise ValueError(f"Provider {custom_llm_provider} not found")
529
+ config_class = create_config_class(provider_config)
530
+ api_base, dynamic_api_key = config_class()._get_openai_compatible_provider_info(
531
+ api_base, api_key
532
+ )
533
+ return model, custom_llm_provider, dynamic_api_key, api_base
534
+
535
+ if custom_llm_provider == "perplexity":
536
+ # perplexity is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.perplexity.ai
537
+ (
538
+ api_base,
539
+ dynamic_api_key,
540
+ ) = litellm.PerplexityChatConfig()._get_openai_compatible_provider_info(
541
+ api_base, api_key
542
+ )
543
+ elif custom_llm_provider == "aiohttp_openai":
544
+ return model, "aiohttp_openai", api_key, api_base
545
+ elif custom_llm_provider == "anyscale":
546
+ # anyscale is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
547
+ api_base = api_base or get_secret_str("ANYSCALE_API_BASE") or "https://api.endpoints.anyscale.com/v1" # type: ignore
548
+ dynamic_api_key = api_key or get_secret_str("ANYSCALE_API_KEY")
549
+ elif custom_llm_provider == "deepinfra":
550
+ (
551
+ api_base,
552
+ dynamic_api_key,
553
+ ) = litellm.DeepInfraConfig()._get_openai_compatible_provider_info(
554
+ api_base, api_key
555
+ )
556
+ elif custom_llm_provider == "empower":
557
+ api_base = (
558
+ api_base
559
+ or get_secret("EMPOWER_API_BASE")
560
+ or "https://app.empower.dev/api/v1"
561
+ ) # type: ignore
562
+ dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY")
563
+ elif custom_llm_provider == "groq":
564
+ (
565
+ api_base,
566
+ dynamic_api_key,
567
+ ) = litellm.GroqChatConfig()._get_openai_compatible_provider_info(
568
+ api_base, api_key
569
+ )
570
+ elif custom_llm_provider == "bedrock_mantle":
571
+ (
572
+ api_base,
573
+ dynamic_api_key,
574
+ ) = litellm.BedrockMantleChatConfig()._get_openai_compatible_provider_info(
575
+ api_base, api_key
576
+ )
577
+ elif custom_llm_provider == "nvidia_nim":
578
+ # nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
579
+ api_base = (
580
+ api_base
581
+ or get_secret("NVIDIA_NIM_API_BASE")
582
+ or "https://integrate.api.nvidia.com/v1"
583
+ ) # type: ignore
584
+ dynamic_api_key = api_key or get_secret_str("NVIDIA_NIM_API_KEY")
585
+ elif custom_llm_provider == "cerebras":
586
+ api_base = (
587
+ api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1"
588
+ ) # type: ignore
589
+ dynamic_api_key = api_key or get_secret_str("CEREBRAS_API_KEY")
590
+ elif custom_llm_provider == "baseten":
591
+ # Use BasetenConfig to determine the appropriate API base URL
592
+ if api_base is None:
593
+ api_base = litellm.BasetenConfig.get_api_base_for_model(model)
594
+ else:
595
+ api_base = (
596
+ api_base
597
+ or get_secret_str("BASETEN_API_BASE")
598
+ or "https://inference.baseten.co/v1"
599
+ )
600
+ dynamic_api_key = api_key or get_secret_str("BASETEN_API_KEY")
601
+ elif custom_llm_provider == "sambanova":
602
+ api_base = (
603
+ api_base
604
+ or get_secret("SAMBANOVA_API_BASE")
605
+ or "https://api.sambanova.ai/v1"
606
+ ) # type: ignore
607
+ dynamic_api_key = api_key or get_secret_str("SAMBANOVA_API_KEY")
608
+ elif custom_llm_provider == "meta_llama":
609
+ api_base = (
610
+ api_base
611
+ or get_secret("LLAMA_API_BASE")
612
+ or "https://api.llama.com/compat/v1"
613
+ ) # type: ignore
614
+ dynamic_api_key = api_key or get_secret_str("LLAMA_API_KEY")
615
+ elif custom_llm_provider == "nebius":
616
+ api_base = (
617
+ api_base
618
+ or get_secret("NEBIUS_API_BASE")
619
+ or "https://api.studio.nebius.ai/v1"
620
+ ) # type: ignore
621
+ dynamic_api_key = api_key or get_secret_str("NEBIUS_API_KEY")
622
+ elif custom_llm_provider == "ollama":
623
+ api_base = (
624
+ api_base or get_secret("OLLAMA_API_BASE") or "http://localhost:11434"
625
+ ) # type: ignore
626
+ dynamic_api_key = api_key or get_secret_str("OLLAMA_API_KEY")
627
+ elif (custom_llm_provider == "ai21_chat") or (
628
+ custom_llm_provider == "ai21" and model in litellm.ai21_chat_models
629
+ ):
630
+ api_base = (
631
+ api_base or get_secret("AI21_API_BASE") or "https://api.ai21.com/studio/v1"
632
+ ) # type: ignore
633
+ dynamic_api_key = api_key or get_secret_str("AI21_API_KEY")
634
+ custom_llm_provider = "ai21_chat"
635
+ elif custom_llm_provider == "volcengine":
636
+ # volcengine is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
637
+ api_base = (
638
+ api_base
639
+ or get_secret("VOLCENGINE_API_BASE")
640
+ or "https://ark.cn-beijing.volces.com/api/v3"
641
+ ) # type: ignore
642
+ dynamic_api_key = api_key or get_secret_str("VOLCENGINE_API_KEY")
643
+ elif custom_llm_provider == "codestral":
644
+ # codestral is openai compatible, we just need to set this to custom_openai and have the api_base be https://codestral.mistral.ai/v1
645
+ api_base = (
646
+ api_base
647
+ or get_secret("CODESTRAL_API_BASE")
648
+ or "https://codestral.mistral.ai/v1"
649
+ ) # type: ignore
650
+ dynamic_api_key = api_key or get_secret_str("CODESTRAL_API_KEY")
651
+ elif custom_llm_provider == "hosted_vllm":
652
+ # vllm is openai compatible, we just need to set this to custom_openai
653
+ (
654
+ api_base,
655
+ dynamic_api_key,
656
+ ) = litellm.HostedVLLMChatConfig()._get_openai_compatible_provider_info(
657
+ api_base, api_key
658
+ )
659
+ elif custom_llm_provider == "llamafile":
660
+ # llamafile is OpenAI compatible.
661
+ (
662
+ api_base,
663
+ dynamic_api_key,
664
+ ) = litellm.LlamafileChatConfig()._get_openai_compatible_provider_info(
665
+ api_base, api_key
666
+ )
667
+ elif custom_llm_provider == "datarobot":
668
+ # DataRobot is OpenAI compatible.
669
+ (
670
+ api_base,
671
+ dynamic_api_key,
672
+ ) = litellm.DataRobotConfig()._get_openai_compatible_provider_info(
673
+ api_base, api_key
674
+ )
675
+ elif custom_llm_provider == "lm_studio":
676
+ # lm_studio is openai compatible, we just need to set this to custom_openai
677
+ (
678
+ api_base,
679
+ dynamic_api_key,
680
+ ) = litellm.LMStudioChatConfig()._get_openai_compatible_provider_info(
681
+ api_base, api_key
682
+ )
683
+ elif custom_llm_provider == "deepseek":
684
+ # deepseek is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.deepseek.com/v1
685
+ api_base = (
686
+ api_base
687
+ or get_secret("DEEPSEEK_API_BASE")
688
+ or "https://api.deepseek.com/beta"
689
+ ) # type: ignore
690
+
691
+ dynamic_api_key = api_key or get_secret_str("DEEPSEEK_API_KEY")
692
+ elif custom_llm_provider == "fireworks_ai":
693
+ # fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.fireworks.ai/inference/v1
694
+ (
695
+ api_base,
696
+ dynamic_api_key,
697
+ ) = litellm.FireworksAIConfig()._get_openai_compatible_provider_info(
698
+ api_base=api_base, api_key=api_key
699
+ )
700
+ elif custom_llm_provider == "azure_ai":
701
+ (
702
+ api_base,
703
+ dynamic_api_key,
704
+ custom_llm_provider,
705
+ ) = litellm.AzureAIStudioConfig()._get_openai_compatible_provider_info(
706
+ model, api_base, api_key, custom_llm_provider
707
+ )
708
+ elif custom_llm_provider == "github":
709
+ api_base = (
710
+ api_base
711
+ or get_secret_str("GITHUB_API_BASE")
712
+ or "https://models.inference.ai.azure.com" # This is github's default base url
713
+ )
714
+ dynamic_api_key = api_key or get_secret_str("GITHUB_API_KEY")
715
+ elif custom_llm_provider == "litellm_proxy":
716
+ (
717
+ api_base,
718
+ dynamic_api_key,
719
+ ) = litellm.LiteLLMProxyChatConfig()._get_openai_compatible_provider_info(
720
+ api_base=api_base, api_key=api_key
721
+ )
722
+
723
+ elif custom_llm_provider == "mistral":
724
+ (
725
+ api_base,
726
+ dynamic_api_key,
727
+ ) = litellm.MistralConfig()._get_openai_compatible_provider_info(
728
+ api_base, api_key
729
+ )
730
+ elif custom_llm_provider == "jina_ai":
731
+ (
732
+ custom_llm_provider,
733
+ api_base,
734
+ dynamic_api_key,
735
+ ) = litellm.JinaAIEmbeddingConfig()._get_openai_compatible_provider_info(
736
+ api_base, api_key
737
+ )
738
+ elif custom_llm_provider == "xai":
739
+ (
740
+ api_base,
741
+ dynamic_api_key,
742
+ ) = litellm.XAIChatConfig()._get_openai_compatible_provider_info(
743
+ api_base, api_key
744
+ )
745
+ elif custom_llm_provider == "zai":
746
+ (
747
+ api_base,
748
+ dynamic_api_key,
749
+ ) = litellm.ZAIChatConfig()._get_openai_compatible_provider_info(
750
+ api_base, api_key
751
+ )
752
+ elif custom_llm_provider == "together_ai":
753
+ api_base = (
754
+ api_base
755
+ or get_secret_str("TOGETHER_AI_API_BASE")
756
+ or "https://api.together.xyz/v1"
757
+ ) # type: ignore
758
+ dynamic_api_key = api_key or (
759
+ get_secret_str("TOGETHER_API_KEY")
760
+ or get_secret_str("TOGETHER_AI_API_KEY")
761
+ or get_secret_str("TOGETHERAI_API_KEY")
762
+ or get_secret_str("TOGETHER_AI_TOKEN")
763
+ )
764
+ elif custom_llm_provider == "friendliai":
765
+ api_base = (
766
+ api_base
767
+ or get_secret("FRIENDLI_API_BASE")
768
+ or "https://api.friendli.ai/serverless/v1"
769
+ ) # type: ignore
770
+ dynamic_api_key = (
771
+ api_key
772
+ or get_secret_str("FRIENDLIAI_API_KEY")
773
+ or get_secret_str("FRIENDLI_TOKEN")
774
+ )
775
+ elif custom_llm_provider == "galadriel":
776
+ api_base = (
777
+ api_base
778
+ or get_secret("GALADRIEL_API_BASE")
779
+ or "https://api.galadriel.com/v1"
780
+ ) # type: ignore
781
+ dynamic_api_key = api_key or get_secret_str("GALADRIEL_API_KEY")
782
+ elif custom_llm_provider == "github_copilot":
783
+ (
784
+ api_base,
785
+ dynamic_api_key,
786
+ custom_llm_provider,
787
+ ) = litellm.GithubCopilotConfig()._get_openai_compatible_provider_info(
788
+ model, api_base, api_key, custom_llm_provider
789
+ )
790
+ elif custom_llm_provider == "chatgpt":
791
+ (
792
+ api_base,
793
+ dynamic_api_key,
794
+ custom_llm_provider,
795
+ ) = litellm.ChatGPTConfig()._get_openai_compatible_provider_info(
796
+ model, api_base, api_key, custom_llm_provider
797
+ )
798
+ elif custom_llm_provider == "novita":
799
+ api_base = (
800
+ api_base
801
+ or get_secret("NOVITA_API_BASE")
802
+ or "https://api.novita.ai/v3/openai"
803
+ ) # type: ignore
804
+ dynamic_api_key = api_key or get_secret_str("NOVITA_API_KEY")
805
+ elif custom_llm_provider == "snowflake":
806
+ (
807
+ api_base,
808
+ dynamic_api_key,
809
+ ) = litellm.SnowflakeConfig()._get_openai_compatible_provider_info(
810
+ api_base, api_key
811
+ )
812
+ elif custom_llm_provider == "gradient_ai":
813
+ (
814
+ api_base,
815
+ dynamic_api_key,
816
+ ) = litellm.GradientAIConfig()._get_openai_compatible_provider_info(
817
+ api_base, api_key
818
+ )
819
+ elif custom_llm_provider == "featherless_ai":
820
+ (
821
+ api_base,
822
+ dynamic_api_key,
823
+ ) = litellm.FeatherlessAIConfig()._get_openai_compatible_provider_info(
824
+ api_base, api_key
825
+ )
826
+ elif custom_llm_provider == "nscale":
827
+ (
828
+ api_base,
829
+ dynamic_api_key,
830
+ ) = litellm.NscaleConfig()._get_openai_compatible_provider_info(
831
+ api_base=api_base, api_key=api_key
832
+ )
833
+ elif custom_llm_provider == "heroku":
834
+ (
835
+ api_base,
836
+ dynamic_api_key,
837
+ ) = litellm.HerokuChatConfig()._get_openai_compatible_provider_info(
838
+ api_base, api_key
839
+ )
840
+ elif custom_llm_provider == "dashscope":
841
+ (
842
+ api_base,
843
+ dynamic_api_key,
844
+ ) = litellm.DashScopeChatConfig()._get_openai_compatible_provider_info(
845
+ api_base, api_key
846
+ )
847
+ elif custom_llm_provider == "moonshot":
848
+ (
849
+ api_base,
850
+ dynamic_api_key,
851
+ ) = litellm.MoonshotChatConfig()._get_openai_compatible_provider_info(
852
+ api_base, api_key
853
+ )
854
+ # publicai is now handled by JSON config (see litellm/llms/openai_like/providers.json)
855
+ elif custom_llm_provider == "docker_model_runner":
856
+ (
857
+ api_base,
858
+ dynamic_api_key,
859
+ ) = litellm.DockerModelRunnerChatConfig()._get_openai_compatible_provider_info(
860
+ api_base, api_key
861
+ )
862
+ elif custom_llm_provider == "v0":
863
+ (
864
+ api_base,
865
+ dynamic_api_key,
866
+ ) = litellm.V0ChatConfig()._get_openai_compatible_provider_info(
867
+ api_base, api_key
868
+ )
869
+ elif custom_llm_provider == "morph":
870
+ (
871
+ api_base,
872
+ dynamic_api_key,
873
+ ) = litellm.MorphChatConfig()._get_openai_compatible_provider_info(
874
+ api_base, api_key
875
+ )
876
+ elif custom_llm_provider == "lambda_ai":
877
+ (
878
+ api_base,
879
+ dynamic_api_key,
880
+ ) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info(
881
+ api_base, api_key
882
+ )
883
+ elif custom_llm_provider == "hyperbolic":
884
+ (
885
+ api_base,
886
+ dynamic_api_key,
887
+ ) = litellm.HyperbolicChatConfig()._get_openai_compatible_provider_info(
888
+ api_base, api_key
889
+ )
890
+ elif custom_llm_provider == "vercel_ai_gateway":
891
+ (
892
+ api_base,
893
+ dynamic_api_key,
894
+ ) = litellm.VercelAIGatewayConfig()._get_openai_compatible_provider_info(
895
+ api_base, api_key
896
+ )
897
+ elif custom_llm_provider == "aiml":
898
+ (
899
+ api_base,
900
+ dynamic_api_key,
901
+ ) = litellm.AIMLChatConfig()._get_openai_compatible_provider_info(
902
+ api_base, api_key
903
+ )
904
+ elif custom_llm_provider == "wandb":
905
+ api_base = (
906
+ api_base
907
+ or get_secret("WANDB_API_BASE")
908
+ or "https://api.inference.wandb.ai/v1"
909
+ ) # type: ignore
910
+ dynamic_api_key = api_key or get_secret_str("WANDB_API_KEY")
911
+ elif custom_llm_provider == "lemonade":
912
+ (
913
+ api_base,
914
+ dynamic_api_key,
915
+ ) = litellm.LemonadeChatConfig()._get_openai_compatible_provider_info(
916
+ api_base, api_key
917
+ )
918
+ elif custom_llm_provider == "clarifai":
919
+ (
920
+ api_base,
921
+ dynamic_api_key,
922
+ ) = litellm.ClarifaiConfig()._get_openai_compatible_provider_info(
923
+ api_base, api_key
924
+ )
925
+ elif custom_llm_provider == "ragflow":
926
+ full_model = f"ragflow/{model}"
927
+ (
928
+ api_base,
929
+ dynamic_api_key,
930
+ _,
931
+ ) = litellm.RAGFlowConfig()._get_openai_compatible_provider_info(
932
+ full_model, api_base, api_key, "ragflow"
933
+ )
934
+ model = full_model
935
+ elif custom_llm_provider == "langgraph":
936
+ # LangGraph is a custom provider, just need to set api_base
937
+ api_base = (
938
+ api_base or get_secret_str("LANGGRAPH_API_BASE") or "http://localhost:2024"
939
+ )
940
+ dynamic_api_key = api_key or get_secret_str("LANGGRAPH_API_KEY")
941
+ elif custom_llm_provider == "manus":
942
+ # Manus is OpenAI compatible for responses API
943
+ api_base = (
944
+ api_base or get_secret_str("MANUS_API_BASE") or "https://api.manus.im"
945
+ )
946
+ dynamic_api_key = api_key or get_secret_str("MANUS_API_KEY")
947
+
948
+ if api_base is not None and not isinstance(api_base, str):
949
+ raise Exception("api base needs to be a string. api_base={}".format(api_base))
950
+ if dynamic_api_key is not None and not isinstance(dynamic_api_key, str):
951
+ raise Exception(
952
+ "dynamic_api_key needs to be a string. dynamic_api_key={}".format(
953
+ dynamic_api_key
954
+ )
955
+ )
956
+ if dynamic_api_key is None and api_key is not None:
957
+ dynamic_api_key = api_key
958
+ return model, custom_llm_provider, dynamic_api_key, api_base
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_model_cost_map.py ADDED
@@ -0,0 +1,299 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Pulls the cost + context window + provider route for known models from https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json
3
+
4
+ This can be disabled by setting the LITELLM_LOCAL_MODEL_COST_MAP environment variable to True.
5
+
6
+ ```
7
+ export LITELLM_LOCAL_MODEL_COST_MAP=True
8
+ ```
9
+ """
10
+
11
+ import json
12
+ import os
13
+ from importlib.resources import files
14
+ from typing import Dict, List, Optional
15
+
16
+ import httpx
17
+
18
+ from litellm import verbose_logger
19
+ from litellm.constants import (
20
+ MODEL_COST_MAP_MAX_SHRINK_RATIO,
21
+ MODEL_COST_MAP_MIN_MODEL_COUNT,
22
+ )
23
+
24
+
25
+ class GetModelCostMap:
26
+ """
27
+ Handles fetching, validating, and loading the model cost map.
28
+
29
+ Only the backup model *count* is cached (a single int). The full
30
+ backup dict is never held in memory — it is only parsed when it
31
+ needs to be *returned* as a fallback.
32
+ """
33
+
34
+ _backup_model_count: int = -1 # -1 = not yet loaded
35
+
36
+ @staticmethod
37
+ def load_local_model_cost_map() -> dict:
38
+ """Load the local backup model cost map bundled with the package."""
39
+ content = json.loads(
40
+ files("litellm")
41
+ .joinpath("model_prices_and_context_window_backup.json")
42
+ .read_text(encoding="utf-8")
43
+ )
44
+ return content
45
+
46
+ @classmethod
47
+ def _get_backup_model_count(cls) -> int:
48
+ """Return the number of models in the local backup (cached int)."""
49
+ if cls._backup_model_count < 0:
50
+ backup = cls.load_local_model_cost_map()
51
+ cls._backup_model_count = len(backup)
52
+ return cls._backup_model_count
53
+
54
+ @staticmethod
55
+ def _check_is_valid_dict(fetched_map: dict) -> bool:
56
+ """Check 1: fetched map is a non-empty dict."""
57
+ if not isinstance(fetched_map, dict):
58
+ verbose_logger.warning(
59
+ "LiteLLM: Fetched model cost map is not a dict (type=%s). "
60
+ "Falling back to local backup.",
61
+ type(fetched_map).__name__,
62
+ )
63
+ return False
64
+
65
+ if len(fetched_map) == 0:
66
+ verbose_logger.warning(
67
+ "LiteLLM: Fetched model cost map is empty. "
68
+ "Falling back to local backup.",
69
+ )
70
+ return False
71
+
72
+ return True
73
+
74
+ @classmethod
75
+ def _check_model_count_not_reduced(
76
+ cls,
77
+ fetched_map: dict,
78
+ backup_model_count: int,
79
+ min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
80
+ max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
81
+ ) -> bool:
82
+ """Check 2: model count has not reduced significantly vs backup."""
83
+ fetched_count = len(fetched_map)
84
+
85
+ if fetched_count < min_model_count:
86
+ verbose_logger.warning(
87
+ "LiteLLM: Fetched model cost map has only %d models (minimum=%d). "
88
+ "This may indicate a corrupted upstream file. "
89
+ "Falling back to local backup.",
90
+ fetched_count,
91
+ min_model_count,
92
+ )
93
+ return False
94
+
95
+ if (
96
+ backup_model_count > 0
97
+ and fetched_count < backup_model_count * max_shrink_ratio
98
+ ):
99
+ verbose_logger.warning(
100
+ "LiteLLM: Fetched model cost map shrank significantly "
101
+ "(fetched=%d, backup=%d, threshold=%.0f%%). "
102
+ "This may indicate a corrupted upstream file. "
103
+ "Falling back to local backup.",
104
+ fetched_count,
105
+ backup_model_count,
106
+ max_shrink_ratio * 100,
107
+ )
108
+ return False
109
+
110
+ return True
111
+
112
+ @classmethod
113
+ def validate_model_cost_map(
114
+ cls,
115
+ fetched_map: dict,
116
+ backup_model_count: int,
117
+ min_model_count: int = MODEL_COST_MAP_MIN_MODEL_COUNT,
118
+ max_shrink_ratio: float = MODEL_COST_MAP_MAX_SHRINK_RATIO,
119
+ ) -> bool:
120
+ """
121
+ Validate the integrity of a fetched model cost map.
122
+
123
+ Runs each check in order and returns False on the first failure.
124
+
125
+ Checks:
126
+ 1. ``_check_is_valid_dict`` -- fetched map is a non-empty dict.
127
+ 2. ``_check_model_count_not_reduced`` -- model count meets minimum
128
+ and has not shrunk >``max_shrink_ratio`` vs backup.
129
+
130
+ Returns True if all checks pass, False otherwise.
131
+ """
132
+ if not cls._check_is_valid_dict(fetched_map):
133
+ return False
134
+
135
+ if not cls._check_model_count_not_reduced(
136
+ fetched_map=fetched_map,
137
+ backup_model_count=backup_model_count,
138
+ min_model_count=min_model_count,
139
+ max_shrink_ratio=max_shrink_ratio,
140
+ ):
141
+ return False
142
+
143
+ return True
144
+
145
+ @staticmethod
146
+ def fetch_remote_model_cost_map(url: str, timeout: int = 5) -> dict:
147
+ """
148
+ Fetch the model cost map from a remote URL.
149
+
150
+ Returns the parsed JSON dict. Raises on network/parse errors
151
+ (caller is expected to handle).
152
+ """
153
+ response = httpx.get(url, timeout=timeout)
154
+ response.raise_for_status()
155
+ return response.json()
156
+
157
+
158
+ class ModelCostMapSourceInfo:
159
+ """Tracks the source of the currently loaded model cost map."""
160
+
161
+ source: str = "local" # "local" or "remote"
162
+ url: Optional[str] = None
163
+ is_env_forced: bool = False
164
+ fallback_reason: Optional[str] = None
165
+
166
+
167
+ # Module-level singleton tracking the source of the current cost map
168
+ _cost_map_source_info = ModelCostMapSourceInfo()
169
+
170
+
171
+ def get_model_cost_map_source_info() -> dict:
172
+ """
173
+ Return metadata about where the current model cost map was loaded from.
174
+
175
+ Returns a dict with:
176
+ - source: "local" or "remote"
177
+ - url: the remote URL attempted (or None for local-only)
178
+ - is_env_forced: True if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage
179
+ - fallback_reason: human-readable reason if remote failed and local was used
180
+ """
181
+ return {
182
+ "source": _cost_map_source_info.source,
183
+ "url": _cost_map_source_info.url,
184
+ "is_env_forced": _cost_map_source_info.is_env_forced,
185
+ "fallback_reason": _cost_map_source_info.fallback_reason,
186
+ }
187
+
188
+
189
+ def _expand_model_aliases(model_cost: dict) -> dict:
190
+ """
191
+ Expand ``aliases`` lists in model cost entries into top-level entries.
192
+
193
+ Each alias gets a reference to the **same** dict object as the canonical
194
+ entry (zero memory overhead). The ``aliases`` key is removed from the
195
+ entry so downstream code never sees it.
196
+
197
+ If an alias collides with an existing canonical entry the alias is
198
+ skipped and a warning is logged.
199
+ """
200
+ aliases_to_add: Dict[str, dict] = {}
201
+ keys_with_aliases: List[str] = []
202
+
203
+ for model_name, model_info in model_cost.items():
204
+ aliases: Optional[list] = model_info.get("aliases")
205
+ if aliases is None:
206
+ continue
207
+ keys_with_aliases.append(model_name)
208
+ if not isinstance(aliases, list):
209
+ verbose_logger.warning(
210
+ "LiteLLM model alias field for '%s' is not a list (got %s) — skipping.",
211
+ model_name,
212
+ type(aliases).__name__,
213
+ )
214
+ continue
215
+ if not aliases:
216
+ continue
217
+ for alias in aliases:
218
+ if alias in model_cost:
219
+ verbose_logger.warning(
220
+ "LiteLLM model alias conflict: alias '%s' (from '%s') "
221
+ "already exists as a canonical entry — skipping.",
222
+ alias,
223
+ model_name,
224
+ )
225
+ continue
226
+ if alias in aliases_to_add:
227
+ verbose_logger.warning(
228
+ "LiteLLM model alias conflict: alias '%s' (from '%s') "
229
+ "was already claimed by another entry — skipping.",
230
+ alias,
231
+ model_name,
232
+ )
233
+ continue
234
+ aliases_to_add[alias] = model_info # same dict reference
235
+
236
+ # Remove the ``aliases`` key from entries so it doesn't pollute model info
237
+ for key in keys_with_aliases:
238
+ model_cost[key].pop("aliases", None)
239
+
240
+ model_cost.update(aliases_to_add)
241
+ return model_cost
242
+
243
+
244
+ def get_model_cost_map(url: str) -> dict:
245
+ """
246
+ Public entry point — returns the model cost map dict.
247
+
248
+ 1. If ``LITELLM_LOCAL_MODEL_COST_MAP`` is set, uses the local backup only.
249
+ 2. Otherwise fetches from ``url``, validates integrity, and falls back
250
+ to the local backup on any failure.
251
+
252
+ Only the backup model count is cached (a single int) for validation.
253
+ The full backup dict is only parsed when it must be *returned* as a
254
+ fallback — it is never held in memory long-term.
255
+ """
256
+ # Note: can't use get_secret_bool here — this runs during litellm.__init__
257
+ # before litellm._key_management_settings is set.
258
+ if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
259
+ _cost_map_source_info.source = "local"
260
+ _cost_map_source_info.url = None
261
+ _cost_map_source_info.is_env_forced = True
262
+ _cost_map_source_info.fallback_reason = None
263
+ return _expand_model_aliases(GetModelCostMap.load_local_model_cost_map())
264
+
265
+ _cost_map_source_info.url = url
266
+ _cost_map_source_info.is_env_forced = False
267
+
268
+ try:
269
+ content = GetModelCostMap.fetch_remote_model_cost_map(url)
270
+ except Exception as e:
271
+ verbose_logger.warning(
272
+ "LiteLLM: Failed to fetch remote model cost map from %s: %s. "
273
+ "Falling back to local backup.",
274
+ url,
275
+ str(e),
276
+ )
277
+ _cost_map_source_info.source = "local"
278
+ _cost_map_source_info.fallback_reason = f"Remote fetch failed: {str(e)}"
279
+ return _expand_model_aliases(GetModelCostMap.load_local_model_cost_map())
280
+
281
+ # Validate using cached count (cheap int comparison, no file I/O)
282
+ if not GetModelCostMap.validate_model_cost_map(
283
+ fetched_map=content,
284
+ backup_model_count=GetModelCostMap._get_backup_model_count(),
285
+ ):
286
+ verbose_logger.warning(
287
+ "LiteLLM: Fetched model cost map failed integrity check. "
288
+ "Using local backup instead. url=%s",
289
+ url,
290
+ )
291
+ _cost_map_source_info.source = "local"
292
+ _cost_map_source_info.fallback_reason = (
293
+ "Remote data failed integrity validation"
294
+ )
295
+ return _expand_model_aliases(GetModelCostMap.load_local_model_cost_map())
296
+
297
+ _cost_map_source_info.source = "remote"
298
+ _cost_map_source_info.fallback_reason = None
299
+ return _expand_model_aliases(content)
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_provider_specific_headers.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Dict, Optional
2
+
3
+ from litellm.types.utils import ProviderSpecificHeader
4
+
5
+
6
+ class ProviderSpecificHeaderUtils:
7
+ @staticmethod
8
+ def get_provider_specific_headers(
9
+ provider_specific_header: Optional[ProviderSpecificHeader],
10
+ custom_llm_provider: Optional[str],
11
+ ) -> Dict:
12
+ """
13
+ Get the provider specific headers for the given custom llm provider.
14
+
15
+ Supports comma-separated provider lists for headers that work across multiple providers.
16
+
17
+ Returns:
18
+ Dict: The provider specific headers for the given custom llm provider
19
+ """
20
+ if provider_specific_header is None or custom_llm_provider is None:
21
+ return {}
22
+
23
+ stored_providers = provider_specific_header.get("custom_llm_provider", "")
24
+ provider_list = [p.strip() for p in stored_providers.split(",")]
25
+
26
+ if custom_llm_provider in provider_list:
27
+ return provider_specific_header.get("extra_headers", {})
28
+
29
+ return {}
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/get_supported_openai_params.py ADDED
@@ -0,0 +1,320 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Literal, Optional
2
+
3
+ import litellm
4
+ from litellm.exceptions import BadRequestError
5
+ from litellm.types.utils import LlmProviders, LlmProvidersSet
6
+
7
+
8
+ def get_supported_openai_params( # noqa: PLR0915
9
+ model: str,
10
+ custom_llm_provider: Optional[str] = None,
11
+ request_type: Literal[
12
+ "chat_completion", "embeddings", "transcription"
13
+ ] = "chat_completion",
14
+ ) -> Optional[list]:
15
+ """
16
+ Returns the supported openai params for a given model + provider
17
+
18
+ Example:
19
+ ```
20
+ get_supported_openai_params(model="anthropic.claude-3", custom_llm_provider="bedrock")
21
+ ```
22
+
23
+ Returns:
24
+ - List if custom_llm_provider is mapped
25
+ - None if unmapped
26
+ """
27
+ if not custom_llm_provider:
28
+ try:
29
+ custom_llm_provider = litellm.get_llm_provider(model=model)[1]
30
+ except BadRequestError:
31
+ return None
32
+
33
+ if custom_llm_provider in LlmProvidersSet:
34
+ provider_config = litellm.ProviderConfigManager.get_provider_chat_config(
35
+ model=model, provider=LlmProviders(custom_llm_provider)
36
+ )
37
+ elif custom_llm_provider.split("/")[0] in LlmProvidersSet:
38
+ provider_config = litellm.ProviderConfigManager.get_provider_chat_config(
39
+ model=model, provider=LlmProviders(custom_llm_provider.split("/")[0])
40
+ )
41
+ else:
42
+ provider_config = None
43
+
44
+ if provider_config and request_type == "chat_completion":
45
+ return provider_config.get_supported_openai_params(model=model)
46
+
47
+ if custom_llm_provider == "bedrock":
48
+ return litellm.AmazonConverseConfig().get_supported_openai_params(model=model)
49
+ elif custom_llm_provider == "meta_llama":
50
+ provider_config = litellm.ProviderConfigManager.get_provider_chat_config(
51
+ model=model, provider=LlmProviders.LLAMA
52
+ )
53
+ if provider_config:
54
+ return provider_config.get_supported_openai_params(model=model)
55
+ elif custom_llm_provider == "ollama":
56
+ return litellm.OllamaConfig().get_supported_openai_params(model=model)
57
+ elif custom_llm_provider == "ollama_chat":
58
+ return litellm.OllamaChatConfig().get_supported_openai_params(model=model)
59
+ elif custom_llm_provider == "anthropic":
60
+ return litellm.AnthropicConfig().get_supported_openai_params(model=model)
61
+ elif custom_llm_provider == "anthropic_text":
62
+ return litellm.AnthropicTextConfig().get_supported_openai_params(model=model)
63
+ elif custom_llm_provider == "fireworks_ai":
64
+ if request_type == "embeddings":
65
+ return litellm.FireworksAIEmbeddingConfig().get_supported_openai_params(
66
+ model=model
67
+ )
68
+ elif request_type == "transcription":
69
+ return litellm.FireworksAIAudioTranscriptionConfig().get_supported_openai_params(
70
+ model=model
71
+ )
72
+ else:
73
+ return litellm.FireworksAIConfig().get_supported_openai_params(model=model)
74
+ elif custom_llm_provider == "nvidia_nim":
75
+ if request_type == "chat_completion":
76
+ return litellm.nvidiaNimConfig.get_supported_openai_params(model=model)
77
+ elif request_type == "embeddings":
78
+ return litellm.nvidiaNimEmbeddingConfig.get_supported_openai_params()
79
+ elif custom_llm_provider == "cerebras":
80
+ return litellm.CerebrasConfig().get_supported_openai_params(model=model)
81
+ elif custom_llm_provider == "baseten":
82
+ return litellm.BasetenConfig().get_supported_openai_params(model=model)
83
+ elif custom_llm_provider == "xai":
84
+ return litellm.XAIChatConfig().get_supported_openai_params(model=model)
85
+ elif custom_llm_provider == "ai21_chat" or custom_llm_provider == "ai21":
86
+ return litellm.AI21ChatConfig().get_supported_openai_params(model=model)
87
+ elif custom_llm_provider == "volcengine":
88
+ return litellm.VolcEngineConfig().get_supported_openai_params(model=model)
89
+ elif custom_llm_provider == "groq":
90
+ return litellm.GroqChatConfig().get_supported_openai_params(model=model)
91
+ elif custom_llm_provider == "bedrock_mantle":
92
+ return litellm.BedrockMantleChatConfig().get_supported_openai_params(
93
+ model=model
94
+ )
95
+ elif custom_llm_provider == "hosted_vllm":
96
+ return litellm.HostedVLLMChatConfig().get_supported_openai_params(model=model)
97
+ elif custom_llm_provider == "vllm":
98
+ return litellm.VLLMConfig().get_supported_openai_params(model=model)
99
+ elif custom_llm_provider == "deepseek":
100
+ return litellm.DeepSeekChatConfig().get_supported_openai_params(model=model)
101
+ elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere":
102
+ return litellm.CohereChatConfig().get_supported_openai_params(model=model)
103
+ elif custom_llm_provider == "maritalk":
104
+ return litellm.MaritalkConfig().get_supported_openai_params(model=model)
105
+ elif custom_llm_provider == "openai":
106
+ if request_type == "transcription":
107
+ transcription_provider_config = (
108
+ litellm.ProviderConfigManager.get_provider_audio_transcription_config(
109
+ model=model, provider=LlmProviders.OPENAI
110
+ )
111
+ )
112
+ if isinstance(
113
+ transcription_provider_config, litellm.OpenAIGPTAudioTranscriptionConfig
114
+ ):
115
+ return transcription_provider_config.get_supported_openai_params(
116
+ model=model
117
+ )
118
+ else:
119
+ raise ValueError(
120
+ f"Unsupported provider config: {transcription_provider_config} for model: {model}"
121
+ )
122
+ return litellm.OpenAIConfig().get_supported_openai_params(model=model)
123
+ elif custom_llm_provider == "sap":
124
+ if request_type == "chat_completion":
125
+ return litellm.GenAIHubOrchestrationConfig().get_supported_openai_params(
126
+ model=model
127
+ )
128
+ elif request_type == "embeddings":
129
+ return litellm.GenAIHubEmbeddingConfig().get_supported_openai_params(
130
+ model=model
131
+ )
132
+ elif custom_llm_provider == "azure":
133
+ if litellm.AzureOpenAIO1Config().is_o_series_model(model=model):
134
+ return litellm.AzureOpenAIO1Config().get_supported_openai_params(
135
+ model=model
136
+ )
137
+ elif litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model=model):
138
+ return litellm.AzureOpenAIGPT5Config().get_supported_openai_params(
139
+ model=model
140
+ )
141
+ else:
142
+ return litellm.AzureOpenAIConfig().get_supported_openai_params(model=model)
143
+ elif custom_llm_provider == "openrouter":
144
+ return litellm.OpenrouterConfig().get_supported_openai_params(model=model)
145
+ elif custom_llm_provider == "vercel_ai_gateway":
146
+ return litellm.VercelAIGatewayConfig().get_supported_openai_params(model=model)
147
+ elif custom_llm_provider == "mistral" or custom_llm_provider == "codestral":
148
+ # mistal and codestral api have the exact same params
149
+ if request_type == "chat_completion":
150
+ return litellm.MistralConfig().get_supported_openai_params(model=model)
151
+ elif request_type == "embeddings":
152
+ return litellm.MistralEmbeddingConfig().get_supported_openai_params()
153
+ elif request_type == "transcription":
154
+ from litellm.llms.mistral.audio_transcription.transformation import (
155
+ MistralAudioTranscriptionConfig,
156
+ )
157
+
158
+ return MistralAudioTranscriptionConfig().get_supported_openai_params(
159
+ model=model
160
+ )
161
+ elif custom_llm_provider == "text-completion-codestral":
162
+ return litellm.CodestralTextCompletionConfig().get_supported_openai_params(
163
+ model=model
164
+ )
165
+ elif custom_llm_provider == "sambanova":
166
+ if request_type == "embeddings":
167
+ litellm.SambaNovaEmbeddingConfig().get_supported_openai_params(model=model)
168
+ else:
169
+ return litellm.SambanovaConfig().get_supported_openai_params(model=model)
170
+ elif custom_llm_provider == "nebius":
171
+ if request_type == "chat_completion":
172
+ return litellm.NebiusConfig().get_supported_openai_params(model=model)
173
+ elif custom_llm_provider == "wandb":
174
+ if request_type == "chat_completion":
175
+ return litellm.WandbConfig().get_supported_openai_params(model=model)
176
+ elif custom_llm_provider == "replicate":
177
+ return litellm.ReplicateConfig().get_supported_openai_params(model=model)
178
+ elif custom_llm_provider == "huggingface":
179
+ return litellm.HuggingFaceChatConfig().get_supported_openai_params(model=model)
180
+ elif custom_llm_provider == "jina_ai":
181
+ if request_type == "embeddings":
182
+ return litellm.JinaAIEmbeddingConfig().get_supported_openai_params(
183
+ model=model
184
+ )
185
+ elif custom_llm_provider == "together_ai":
186
+ return litellm.TogetherAIConfig().get_supported_openai_params(model=model)
187
+ elif custom_llm_provider == "databricks":
188
+ if request_type == "chat_completion":
189
+ return litellm.DatabricksConfig().get_supported_openai_params(model=model)
190
+ elif request_type == "embeddings":
191
+ return litellm.DatabricksEmbeddingConfig().get_supported_openai_params()
192
+ elif custom_llm_provider == "palm" or custom_llm_provider == "gemini":
193
+ return litellm.GoogleAIStudioGeminiConfig().get_supported_openai_params(
194
+ model=model
195
+ )
196
+ elif custom_llm_provider == "novita":
197
+ return litellm.NovitaConfig().get_supported_openai_params(model=model)
198
+ elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta":
199
+ if request_type == "chat_completion":
200
+ if model.startswith("mistral"):
201
+ return litellm.MistralConfig().get_supported_openai_params(model=model)
202
+ elif model.startswith("codestral"):
203
+ return (
204
+ litellm.CodestralTextCompletionConfig().get_supported_openai_params(
205
+ model=model
206
+ )
207
+ )
208
+ elif model.startswith("claude"):
209
+ return litellm.VertexAIAnthropicConfig().get_supported_openai_params(
210
+ model=model
211
+ )
212
+ elif model.startswith("gemini"):
213
+ return litellm.VertexGeminiConfig().get_supported_openai_params(
214
+ model=model
215
+ )
216
+ else:
217
+ return litellm.VertexAILlama3Config().get_supported_openai_params(
218
+ model=model
219
+ )
220
+ elif request_type == "embeddings":
221
+ return litellm.VertexAITextEmbeddingConfig().get_supported_openai_params()
222
+ elif custom_llm_provider == "sagemaker":
223
+ return litellm.SagemakerConfig().get_supported_openai_params(model=model)
224
+ elif custom_llm_provider == "aleph_alpha":
225
+ return [
226
+ "max_tokens",
227
+ "stream",
228
+ "top_p",
229
+ "temperature",
230
+ "presence_penalty",
231
+ "frequency_penalty",
232
+ "n",
233
+ "stop",
234
+ ]
235
+ elif custom_llm_provider == "cloudflare":
236
+ return litellm.CloudflareChatConfig().get_supported_openai_params(model=model)
237
+ elif custom_llm_provider == "nlp_cloud":
238
+ return litellm.NLPCloudConfig().get_supported_openai_params(model=model)
239
+ elif custom_llm_provider == "petals":
240
+ return litellm.PetalsConfig().get_supported_openai_params(model=model)
241
+ elif custom_llm_provider == "deepinfra":
242
+ return litellm.DeepInfraConfig().get_supported_openai_params(model=model)
243
+ elif custom_llm_provider == "perplexity":
244
+ return litellm.PerplexityChatConfig().get_supported_openai_params(model=model)
245
+ elif custom_llm_provider == "nscale":
246
+ return litellm.NscaleConfig().get_supported_openai_params(model=model)
247
+ elif custom_llm_provider == "anyscale":
248
+ return [
249
+ "temperature",
250
+ "top_p",
251
+ "stream",
252
+ "max_tokens",
253
+ "stop",
254
+ "frequency_penalty",
255
+ "presence_penalty",
256
+ ]
257
+ elif custom_llm_provider == "watsonx":
258
+ return litellm.IBMWatsonXChatConfig().get_supported_openai_params(model=model)
259
+ elif custom_llm_provider == "watsonx_text":
260
+ return litellm.IBMWatsonXAIConfig().get_supported_openai_params(model=model)
261
+ elif (
262
+ custom_llm_provider == "custom_openai"
263
+ or custom_llm_provider == "text-completion-openai"
264
+ ):
265
+ return litellm.OpenAITextCompletionConfig().get_supported_openai_params(
266
+ model=model
267
+ )
268
+ elif custom_llm_provider == "predibase":
269
+ return litellm.PredibaseConfig().get_supported_openai_params(model=model)
270
+ elif custom_llm_provider == "voyage":
271
+ return litellm.VoyageEmbeddingConfig().get_supported_openai_params(model=model)
272
+ elif custom_llm_provider == "infinity":
273
+ return litellm.InfinityEmbeddingConfig().get_supported_openai_params(
274
+ model=model
275
+ )
276
+ elif custom_llm_provider == "triton":
277
+ if request_type == "embeddings":
278
+ return litellm.TritonEmbeddingConfig().get_supported_openai_params(
279
+ model=model
280
+ )
281
+ else:
282
+ return litellm.TritonConfig().get_supported_openai_params(model=model)
283
+ elif custom_llm_provider == "deepgram":
284
+ if request_type == "transcription":
285
+ return (
286
+ litellm.DeepgramAudioTranscriptionConfig().get_supported_openai_params(
287
+ model=model
288
+ )
289
+ )
290
+ elif custom_llm_provider == "ovhcloud":
291
+ if request_type == "transcription":
292
+ from litellm.llms.ovhcloud.audio_transcription.transformation import (
293
+ OVHCloudAudioTranscriptionConfig,
294
+ )
295
+
296
+ return OVHCloudAudioTranscriptionConfig().get_supported_openai_params(
297
+ model=model
298
+ )
299
+ elif custom_llm_provider == "elevenlabs":
300
+ if request_type == "transcription":
301
+ from litellm.llms.elevenlabs.audio_transcription.transformation import (
302
+ ElevenLabsAudioTranscriptionConfig,
303
+ )
304
+
305
+ return ElevenLabsAudioTranscriptionConfig().get_supported_openai_params(
306
+ model=model
307
+ )
308
+ elif custom_llm_provider in litellm._custom_providers:
309
+ if request_type == "chat_completion":
310
+ provider_config = litellm.ProviderConfigManager.get_provider_chat_config(
311
+ model=model, provider=LlmProviders.CUSTOM
312
+ )
313
+ if provider_config:
314
+ return provider_config.get_supported_openai_params(model=model)
315
+ elif request_type == "embeddings":
316
+ return None
317
+ elif request_type == "transcription":
318
+ return None
319
+
320
+ return None
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/health_check_helpers.py ADDED
@@ -0,0 +1,219 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Helper functions for health check calls.
3
+ """
4
+
5
+ from typing import TYPE_CHECKING, Callable, Dict, Literal, Optional
6
+
7
+ from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS
8
+
9
+ if TYPE_CHECKING:
10
+ from litellm.litellm_core_utils.litellm_logging import Logging
11
+
12
+ # Minimal PDF for health checks - base64 encoded 1-page PDF with just "test"
13
+ TEST_PDF_URL = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="
14
+
15
+
16
+ class HealthCheckHelpers:
17
+ @staticmethod
18
+ async def ahealth_check_wildcard_models(
19
+ model: str,
20
+ custom_llm_provider: str,
21
+ model_params: dict,
22
+ litellm_logging_obj: "Logging",
23
+ ) -> dict:
24
+ from litellm import acompletion
25
+ from litellm.litellm_core_utils.llm_request_utils import (
26
+ pick_cheapest_chat_models_from_llm_provider,
27
+ )
28
+
29
+ # this is a wildcard model, we need to pick a random model from the provider
30
+ cheapest_models = pick_cheapest_chat_models_from_llm_provider(
31
+ custom_llm_provider=custom_llm_provider, n=3
32
+ )
33
+ if len(cheapest_models) == 0:
34
+ raise Exception(
35
+ f"Unable to health check wildcard model for provider {custom_llm_provider}. Add a model on your config.yaml or contribute here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
36
+ )
37
+ if len(cheapest_models) > 1:
38
+ fallback_models = cheapest_models[
39
+ 1:
40
+ ] # Pick the last 2 models from the shuffled list
41
+ else:
42
+ fallback_models = None
43
+ model_params["model"] = cheapest_models[0]
44
+ model_params["litellm_logging_obj"] = litellm_logging_obj
45
+ model_params["fallbacks"] = fallback_models
46
+ model_params["max_tokens"] = model_params.get(
47
+ "max_tokens", 10
48
+ ) # gpt-5-nano throws errors for max_tokens=1
49
+ await acompletion(**model_params)
50
+ return {}
51
+
52
+ @staticmethod
53
+ def _update_model_params_with_health_check_tracking_information(
54
+ model_params: dict,
55
+ ) -> dict:
56
+ """
57
+ Updates the health check model params with tracking information.
58
+
59
+ The following is added at this stage:
60
+ 1. `tags`: This helps identify health check calls in the DB.
61
+ 2. `user_api_key_auth`: This helps identify health check calls in the DB.
62
+ We need this since the DB requires an API Key to track a log in the SpendLogs Table
63
+ """
64
+ from litellm.proxy._types import UserAPIKeyAuth
65
+ from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
66
+
67
+ _metadata_variable_name = "litellm_metadata"
68
+ litellm_metadata = HealthCheckHelpers._get_metadata_for_health_check_call()
69
+ model_params[_metadata_variable_name] = litellm_metadata
70
+ model_params = LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
71
+ data=model_params,
72
+ user_api_key_dict=UserAPIKeyAuth.get_litellm_internal_health_check_user_api_key_auth(),
73
+ _metadata_variable_name=_metadata_variable_name,
74
+ )
75
+ return model_params
76
+
77
+ @staticmethod
78
+ def _get_metadata_for_health_check_call():
79
+ """
80
+ Returns the metadata for the health check call.
81
+ """
82
+ from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME
83
+
84
+ return {
85
+ "tags": [LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME],
86
+ }
87
+
88
+ @staticmethod
89
+ async def _batch_health_check(
90
+ custom_llm_provider: str,
91
+ model_params: dict,
92
+ filtered_model_params: dict,
93
+ ) -> dict:
94
+ """
95
+ Health check for batch mode.
96
+
97
+ Calls list_batches for providers that support it (openai, hosted_vllm, azure,
98
+ vertex_ai). For all other providers (e.g. bedrock) the batch API surface doesn't
99
+ include list_batches, so we fall back to acompletion to verify connectivity and
100
+ credential validity instead.
101
+ """
102
+ import litellm
103
+
104
+ if custom_llm_provider in LIST_BATCHES_SUPPORTED_PROVIDERS:
105
+ return await litellm.alist_batches(**filtered_model_params)
106
+ else:
107
+ return await litellm.acompletion(**model_params)
108
+
109
+ @staticmethod
110
+ def get_mode_handlers(
111
+ model: str,
112
+ custom_llm_provider: str,
113
+ model_params: dict,
114
+ prompt: Optional[str] = None,
115
+ input: Optional[list] = None,
116
+ ) -> Dict[
117
+ Literal[
118
+ "chat",
119
+ "completion",
120
+ "embedding",
121
+ "audio_speech",
122
+ "audio_transcription",
123
+ "image_generation",
124
+ "video_generation",
125
+ "rerank",
126
+ "realtime",
127
+ "batch",
128
+ "responses",
129
+ "ocr",
130
+ ],
131
+ Callable,
132
+ ]:
133
+ """
134
+ Returns a dictionary of mode handlers for health check calls.
135
+
136
+ Mode Handlers are Callables that need to be run for execution of the health check call.
137
+
138
+ Args:
139
+ model: The model name
140
+ custom_llm_provider: The LLM provider
141
+ model_params: The model parameters
142
+ prompt: Optional prompt for health check
143
+ input: Optional input for health check
144
+
145
+ Returns:
146
+ Dictionary mapping mode names to their handler functions
147
+ """
148
+ import litellm
149
+ from litellm.litellm_core_utils.audio_utils.utils import (
150
+ get_audio_file_for_health_check,
151
+ )
152
+ from litellm.litellm_core_utils.health_check_utils import _filter_model_params
153
+ from litellm.realtime_api.main import _realtime_health_check
154
+
155
+ return {
156
+ "chat": lambda: litellm.acompletion(
157
+ **model_params,
158
+ ),
159
+ "completion": lambda: litellm.atext_completion(
160
+ **_filter_model_params(model_params=model_params),
161
+ prompt=prompt or "test",
162
+ ),
163
+ "embedding": lambda: litellm.aembedding(
164
+ **_filter_model_params(model_params=model_params),
165
+ input=input or ["test"],
166
+ ),
167
+ "audio_speech": lambda: litellm.aspeech(
168
+ **{
169
+ **_filter_model_params(model_params=model_params),
170
+ **(
171
+ {"voice": "alloy"}
172
+ if "voice"
173
+ not in _filter_model_params(model_params=model_params)
174
+ else {}
175
+ ),
176
+ },
177
+ input=prompt or "test",
178
+ ),
179
+ "audio_transcription": lambda: litellm.atranscription(
180
+ **_filter_model_params(model_params=model_params),
181
+ file=get_audio_file_for_health_check(),
182
+ ),
183
+ "image_generation": lambda: litellm.aimage_generation(
184
+ **_filter_model_params(model_params=model_params),
185
+ prompt=prompt,
186
+ ),
187
+ "video_generation": lambda: litellm.avideo_generation(
188
+ **_filter_model_params(model_params=model_params),
189
+ prompt=prompt or "test video generation",
190
+ ),
191
+ "rerank": lambda: litellm.arerank(
192
+ **_filter_model_params(model_params=model_params),
193
+ query=prompt or "",
194
+ documents=["my sample text"],
195
+ ),
196
+ "realtime": lambda: _realtime_health_check(
197
+ model=model,
198
+ custom_llm_provider=custom_llm_provider,
199
+ api_base=model_params.get("api_base", None),
200
+ api_key=model_params.get("api_key", None),
201
+ api_version=model_params.get("api_version", None),
202
+ ),
203
+ "batch": lambda: HealthCheckHelpers._batch_health_check(
204
+ custom_llm_provider=custom_llm_provider,
205
+ model_params=model_params,
206
+ filtered_model_params=_filter_model_params(model_params=model_params),
207
+ ),
208
+ "responses": lambda: litellm.aresponses(
209
+ **_filter_model_params(model_params=model_params),
210
+ input=prompt or "test",
211
+ ),
212
+ "ocr": lambda: litellm.aocr(
213
+ **_filter_model_params(model_params=model_params),
214
+ document={
215
+ "type": "document_url",
216
+ "document_url": TEST_PDF_URL,
217
+ },
218
+ ),
219
+ }
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/health_check_utils.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Utils used for litellm.ahealth_check()
3
+ """
4
+
5
+
6
+ def _filter_model_params(model_params: dict) -> dict:
7
+ """Remove 'messages' param from model params."""
8
+ return {k: v for k, v in model_params.items() if k != "messages"}
9
+
10
+
11
+ def _create_health_check_response(response_headers: dict) -> dict:
12
+ response = {}
13
+
14
+ if (
15
+ response_headers.get("x-ratelimit-remaining-requests", None) is not None
16
+ ): # not provided for dall-e requests
17
+ response["x-ratelimit-remaining-requests"] = response_headers[
18
+ "x-ratelimit-remaining-requests"
19
+ ]
20
+
21
+ if response_headers.get("x-ratelimit-remaining-tokens", None) is not None:
22
+ response["x-ratelimit-remaining-tokens"] = response_headers[
23
+ "x-ratelimit-remaining-tokens"
24
+ ]
25
+
26
+ if response_headers.get("x-ms-region", None) is not None:
27
+ response["x-ms-region"] = response_headers["x-ms-region"]
28
+ return response
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/initialize_dynamic_callback_params.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Dict, Optional
2
+ from litellm.secret_managers.main import get_secret_str
3
+ from litellm.types.utils import StandardCallbackDynamicParams
4
+
5
+ # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
6
+ _supported_callback_params = [
7
+ "langfuse_public_key",
8
+ "langfuse_secret",
9
+ "langfuse_secret_key",
10
+ "langfuse_host",
11
+ "langfuse_prompt_version",
12
+ "gcs_bucket_name",
13
+ "gcs_path_service_account",
14
+ "langsmith_api_key",
15
+ "langsmith_project",
16
+ "langsmith_base_url",
17
+ "langsmith_sampling_rate",
18
+ "langsmith_tenant_id",
19
+ "humanloop_api_key",
20
+ "arize_api_key",
21
+ "arize_space_key",
22
+ "arize_space_id",
23
+ "posthog_api_key",
24
+ "posthog_host",
25
+ "braintrust_api_key",
26
+ "braintrust_project",
27
+ "braintrust_host",
28
+ "slack_webhook_url",
29
+ "lunary_public_key",
30
+ "turn_off_message_logging",
31
+ ]
32
+
33
+
34
+ def initialize_standard_callback_dynamic_params(
35
+ kwargs: Optional[Dict] = None,
36
+ ) -> StandardCallbackDynamicParams:
37
+ """
38
+ Initialize the standard callback dynamic params from the kwargs
39
+
40
+ checks if langfuse_secret_key, gcs_bucket_name in kwargs and sets the corresponding attributes in StandardCallbackDynamicParams
41
+ """
42
+
43
+ standard_callback_dynamic_params = StandardCallbackDynamicParams()
44
+ if kwargs:
45
+ # 1. Check top-level kwargs
46
+ for param in _supported_callback_params:
47
+ if param in kwargs:
48
+ _param_value = kwargs.get(param)
49
+ if (
50
+ _param_value is not None
51
+ and isinstance(_param_value, str)
52
+ and "os.environ/" in _param_value
53
+ ):
54
+ _param_value = get_secret_str(secret_name=_param_value)
55
+ standard_callback_dynamic_params[param] = _param_value # type: ignore
56
+
57
+ # 2. Fallback: check "metadata" or "litellm_params" -> "metadata"
58
+ metadata = (kwargs.get("metadata") or {}).copy()
59
+ litellm_params = kwargs.get("litellm_params") or {}
60
+ if isinstance(litellm_params, dict):
61
+ metadata.update(litellm_params.get("metadata") or {})
62
+
63
+ if isinstance(metadata, dict):
64
+ for param in _supported_callback_params:
65
+ if param not in standard_callback_dynamic_params and param in metadata:
66
+ _param_value = metadata.get(param)
67
+ if (
68
+ _param_value is not None
69
+ and isinstance(_param_value, str)
70
+ and "os.environ/" in _param_value
71
+ ):
72
+ _param_value = get_secret_str(secret_name=_param_value)
73
+ standard_callback_dynamic_params[param] = _param_value # type: ignore
74
+
75
+ return standard_callback_dynamic_params
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/json_validation_rule.py ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from typing import Any, Dict, List, Union
3
+
4
+ from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
5
+
6
+
7
+ def normalize_json_schema_types(
8
+ schema: Union[Dict[str, Any], List[Any], Any],
9
+ depth: int = 0,
10
+ max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
11
+ ) -> Union[Dict[str, Any], List[Any], Any]:
12
+ """
13
+ Normalize JSON schema types from uppercase to lowercase format.
14
+
15
+ Some providers (like certain Google services) use uppercase types like 'BOOLEAN', 'STRING', 'ARRAY', 'OBJECT'
16
+ but standard JSON Schema requires lowercase: 'boolean', 'string', 'array', 'object'
17
+
18
+ This function recursively normalizes all type fields in a schema to lowercase.
19
+
20
+ Args:
21
+ schema: The schema to normalize (dict, list, or other)
22
+ depth: Current recursion depth
23
+ max_depth: Maximum recursion depth to prevent infinite loops
24
+
25
+ Returns:
26
+ The normalized schema with lowercase types
27
+ """
28
+ # Prevent infinite recursion
29
+ if depth >= max_depth:
30
+ return schema
31
+
32
+ if not isinstance(schema, (dict, list)):
33
+ return schema
34
+
35
+ # Type mapping from uppercase to lowercase
36
+ type_mapping = {
37
+ "BOOLEAN": "boolean",
38
+ "STRING": "string",
39
+ "ARRAY": "array",
40
+ "OBJECT": "object",
41
+ "NUMBER": "number",
42
+ "INTEGER": "integer",
43
+ "NULL": "null",
44
+ }
45
+
46
+ if isinstance(schema, list):
47
+ return [
48
+ normalize_json_schema_types(item, depth + 1, max_depth) for item in schema
49
+ ]
50
+
51
+ if isinstance(schema, dict):
52
+ normalized_schema: Dict[str, Any] = {}
53
+
54
+ for key, value in schema.items():
55
+ if key == "type" and isinstance(value, str) and value in type_mapping:
56
+ normalized_schema[key] = type_mapping[value]
57
+ elif key == "properties" and isinstance(value, dict):
58
+ # Recursively normalize properties
59
+ normalized_schema[key] = {
60
+ prop_key: normalize_json_schema_types(
61
+ prop_value, depth + 1, max_depth
62
+ )
63
+ for prop_key, prop_value in value.items()
64
+ }
65
+ elif key == "items" and isinstance(value, (dict, list)):
66
+ # Recursively normalize array items
67
+ normalized_schema[key] = normalize_json_schema_types(
68
+ value, depth + 1, max_depth
69
+ )
70
+ elif isinstance(value, (dict, list)):
71
+ # Recursively normalize any nested dict or list
72
+ normalized_schema[key] = normalize_json_schema_types(
73
+ value, depth + 1, max_depth
74
+ )
75
+ else:
76
+ normalized_schema[key] = value
77
+
78
+ return normalized_schema
79
+
80
+ return schema
81
+
82
+
83
+ def normalize_tool_schema(tool: Dict[str, Any]) -> Dict[str, Any]:
84
+ """
85
+ Normalize a tool's parameter schema to use standard JSON Schema lowercase types.
86
+
87
+ Args:
88
+ tool: The tool definition containing function parameters
89
+
90
+ Returns:
91
+ The tool with normalized schema types
92
+ """
93
+ if not isinstance(tool, dict):
94
+ return tool
95
+
96
+ normalized_tool = tool.copy()
97
+
98
+ # Normalize function parameters if present
99
+ if "function" in tool and isinstance(tool["function"], dict):
100
+ normalized_tool["function"] = tool["function"].copy()
101
+ if "parameters" in tool["function"]:
102
+ normalized_tool["function"]["parameters"] = normalize_json_schema_types(
103
+ tool["function"]["parameters"]
104
+ )
105
+
106
+ return normalized_tool
107
+
108
+
109
+ def validate_schema(schema: dict, response: str):
110
+ """
111
+ Validate if the returned json response follows the schema.
112
+
113
+ Params:
114
+ - schema - dict: JSON schema
115
+ - response - str: Received json response as string.
116
+ """
117
+ from jsonschema import ValidationError, validate
118
+
119
+ from litellm import JSONSchemaValidationError
120
+
121
+ try:
122
+ response_dict = json.loads(response)
123
+ except json.JSONDecodeError:
124
+ raise JSONSchemaValidationError(
125
+ model="", llm_provider="", raw_response=response, schema=json.dumps(schema)
126
+ )
127
+
128
+ try:
129
+ validate(response_dict, schema=schema)
130
+ except ValidationError:
131
+ raise JSONSchemaValidationError(
132
+ model="", llm_provider="", raw_response=response, schema=json.dumps(schema)
133
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/litellm_logging.py ADDED
The diff for this file is too large to render. See raw diff
 
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py ADDED
@@ -0,0 +1,729 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Helper utilities for tracking the cost of built-in tools.
3
+ """
4
+
5
+ from typing import Any, Dict, List, Literal, Optional, Tuple
6
+
7
+ import litellm
8
+ from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
9
+ from litellm.types.llms.openai import (
10
+ FileSearchTool,
11
+ ResponsesAPIResponse,
12
+ WebSearchOptions,
13
+ )
14
+ from litellm.types.utils import (
15
+ Message,
16
+ ModelInfo,
17
+ ModelResponse,
18
+ SearchContextCostPerQuery,
19
+ StandardBuiltInToolsParams,
20
+ Usage,
21
+ )
22
+
23
+
24
+ class StandardBuiltInToolCostTracking:
25
+ """
26
+ Helper class for tracking the cost of built-in tools
27
+
28
+ Example: Web Search
29
+ """
30
+
31
+ @staticmethod
32
+ def get_cost_for_built_in_tools(
33
+ model: str,
34
+ response_object: Any,
35
+ usage: Optional[Usage] = None,
36
+ custom_llm_provider: Optional[str] = None,
37
+ standard_built_in_tools_params: Optional[StandardBuiltInToolsParams] = None,
38
+ ) -> float:
39
+ """
40
+ Get the cost of using built-in tools.
41
+
42
+ Supported tools:
43
+ - Web Search
44
+ - File Search
45
+ - Vector Store (Azure)
46
+ - Computer Use (Azure)
47
+ - Code Interpreter (Azure)
48
+ """
49
+ standard_built_in_tools_params = standard_built_in_tools_params or {}
50
+
51
+ # Handle web search
52
+ if StandardBuiltInToolCostTracking.response_object_includes_web_search_call(
53
+ response_object=response_object, usage=usage
54
+ ):
55
+ return StandardBuiltInToolCostTracking._handle_web_search_cost(
56
+ model=model,
57
+ custom_llm_provider=custom_llm_provider,
58
+ usage=usage,
59
+ standard_built_in_tools_params=standard_built_in_tools_params,
60
+ )
61
+
62
+ # Handle file search
63
+ if StandardBuiltInToolCostTracking.response_object_includes_file_search_call(
64
+ response_object=response_object
65
+ ):
66
+ return StandardBuiltInToolCostTracking._handle_file_search_cost(
67
+ model=model,
68
+ custom_llm_provider=custom_llm_provider,
69
+ standard_built_in_tools_params=standard_built_in_tools_params,
70
+ )
71
+
72
+ # Handle Azure assistant features
73
+ return StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
74
+ model=model,
75
+ custom_llm_provider=custom_llm_provider,
76
+ standard_built_in_tools_params=standard_built_in_tools_params,
77
+ )
78
+
79
+ @staticmethod
80
+ def _handle_web_search_cost(
81
+ model: str,
82
+ custom_llm_provider: Optional[str],
83
+ usage: Optional[Usage],
84
+ standard_built_in_tools_params: StandardBuiltInToolsParams,
85
+ ) -> float:
86
+ """Handle web search cost calculation."""
87
+ from litellm.llms import get_cost_for_web_search_request
88
+
89
+ model_info = StandardBuiltInToolCostTracking._safe_get_model_info(
90
+ model=model, custom_llm_provider=custom_llm_provider
91
+ )
92
+
93
+ if custom_llm_provider is None and model_info is not None:
94
+ custom_llm_provider = model_info["litellm_provider"]
95
+
96
+ if (
97
+ model_info is not None
98
+ and usage is not None
99
+ and custom_llm_provider is not None
100
+ ):
101
+ result = get_cost_for_web_search_request(
102
+ custom_llm_provider=custom_llm_provider,
103
+ usage=usage,
104
+ model_info=model_info,
105
+ )
106
+ if result is not None:
107
+ return result
108
+
109
+ return StandardBuiltInToolCostTracking.get_cost_for_web_search(
110
+ web_search_options=standard_built_in_tools_params.get(
111
+ "web_search_options", None
112
+ ),
113
+ model_info=model_info,
114
+ )
115
+
116
+ @staticmethod
117
+ def _handle_file_search_cost(
118
+ model: str,
119
+ custom_llm_provider: Optional[str],
120
+ standard_built_in_tools_params: StandardBuiltInToolsParams,
121
+ ) -> float:
122
+ """Handle file search cost calculation."""
123
+ model_info = StandardBuiltInToolCostTracking._safe_get_model_info(
124
+ model=model, custom_llm_provider=custom_llm_provider
125
+ )
126
+ file_search_raw: Any = standard_built_in_tools_params.get("file_search", {})
127
+ file_search_usage: Optional[FileSearchTool] = (
128
+ FileSearchTool(**file_search_raw) if file_search_raw else None
129
+ )
130
+
131
+ # Convert model_info to dict and extract usage parameters
132
+ model_info_dict = dict(model_info) if model_info is not None else None
133
+ storage_gb, days = StandardBuiltInToolCostTracking._extract_file_search_params(
134
+ file_search_usage
135
+ )
136
+
137
+ return StandardBuiltInToolCostTracking.get_cost_for_file_search(
138
+ file_search=file_search_usage,
139
+ provider=custom_llm_provider,
140
+ model_info=model_info_dict,
141
+ storage_gb=storage_gb,
142
+ days=days,
143
+ )
144
+
145
+ @staticmethod
146
+ def _handle_azure_assistant_costs(
147
+ model: str,
148
+ custom_llm_provider: Optional[str],
149
+ standard_built_in_tools_params: StandardBuiltInToolsParams,
150
+ ) -> float:
151
+ """Handle Azure assistant features cost calculation."""
152
+ if custom_llm_provider != "azure":
153
+ return 0.0
154
+
155
+ model_info = StandardBuiltInToolCostTracking._safe_get_model_info(
156
+ model=model, custom_llm_provider=custom_llm_provider
157
+ )
158
+
159
+ total_cost = 0.0
160
+ total_cost += StandardBuiltInToolCostTracking._get_vector_store_cost(
161
+ model_info, custom_llm_provider, standard_built_in_tools_params
162
+ )
163
+ total_cost += StandardBuiltInToolCostTracking._get_computer_use_cost(
164
+ model_info, custom_llm_provider, standard_built_in_tools_params
165
+ )
166
+ total_cost += StandardBuiltInToolCostTracking._get_code_interpreter_cost(
167
+ model_info, custom_llm_provider, standard_built_in_tools_params
168
+ )
169
+
170
+ return total_cost
171
+
172
+ @staticmethod
173
+ def _extract_file_search_params(
174
+ file_search_usage: Any,
175
+ ) -> Tuple[Optional[float], Optional[float]]:
176
+ """Extract and convert file search parameters safely."""
177
+ storage_gb = None
178
+ days = None
179
+
180
+ if isinstance(file_search_usage, dict):
181
+ storage_gb_val = file_search_usage.get("storage_gb")
182
+ days_val = file_search_usage.get("days")
183
+
184
+ if storage_gb_val is not None:
185
+ try:
186
+ storage_gb = float(storage_gb_val) # type: ignore
187
+ except (TypeError, ValueError):
188
+ storage_gb = None
189
+
190
+ if days_val is not None:
191
+ try:
192
+ days = float(days_val) # type: ignore
193
+ except (TypeError, ValueError):
194
+ days = None
195
+
196
+ return storage_gb, days
197
+
198
+ @staticmethod
199
+ def _get_vector_store_cost(
200
+ model_info: Optional[ModelInfo],
201
+ custom_llm_provider: Optional[str],
202
+ standard_built_in_tools_params: StandardBuiltInToolsParams,
203
+ ) -> float:
204
+ """Calculate vector store cost."""
205
+ vector_store_usage = standard_built_in_tools_params.get(
206
+ "vector_store_usage", None
207
+ )
208
+ if not vector_store_usage:
209
+ return 0.0
210
+
211
+ model_info_dict = dict(model_info) if model_info is not None else None
212
+ vector_store_dict = (
213
+ vector_store_usage if isinstance(vector_store_usage, dict) else {}
214
+ )
215
+
216
+ return StandardBuiltInToolCostTracking.get_cost_for_vector_store(
217
+ vector_store_usage=vector_store_dict,
218
+ provider=custom_llm_provider,
219
+ model_info=model_info_dict,
220
+ )
221
+
222
+ @staticmethod
223
+ def _get_computer_use_cost(
224
+ model_info: Optional[ModelInfo],
225
+ custom_llm_provider: Optional[str],
226
+ standard_built_in_tools_params: StandardBuiltInToolsParams,
227
+ ) -> float:
228
+ """Calculate computer use cost."""
229
+ computer_use_usage = standard_built_in_tools_params.get(
230
+ "computer_use_usage", {}
231
+ )
232
+ if not computer_use_usage:
233
+ return 0.0
234
+
235
+ model_info_dict = dict(model_info) if model_info is not None else None
236
+ (
237
+ input_tokens,
238
+ output_tokens,
239
+ ) = StandardBuiltInToolCostTracking._extract_token_counts(computer_use_usage)
240
+
241
+ return StandardBuiltInToolCostTracking.get_cost_for_computer_use(
242
+ input_tokens=input_tokens,
243
+ output_tokens=output_tokens,
244
+ provider=custom_llm_provider,
245
+ model_info=model_info_dict,
246
+ )
247
+
248
+ @staticmethod
249
+ def _get_code_interpreter_cost(
250
+ model_info: Optional[ModelInfo],
251
+ custom_llm_provider: Optional[str],
252
+ standard_built_in_tools_params: StandardBuiltInToolsParams,
253
+ ) -> float:
254
+ """Calculate code interpreter cost."""
255
+ code_interpreter_sessions = standard_built_in_tools_params.get(
256
+ "code_interpreter_sessions", None
257
+ )
258
+ if not code_interpreter_sessions:
259
+ return 0.0
260
+
261
+ model_info_dict = dict(model_info) if model_info is not None else None
262
+ sessions = StandardBuiltInToolCostTracking._safe_convert_to_int(
263
+ code_interpreter_sessions
264
+ )
265
+
266
+ return StandardBuiltInToolCostTracking.get_cost_for_code_interpreter(
267
+ sessions=sessions,
268
+ provider=custom_llm_provider,
269
+ model_info=model_info_dict,
270
+ )
271
+
272
+ @staticmethod
273
+ def _extract_token_counts(
274
+ computer_use_usage: Any,
275
+ ) -> Tuple[Optional[int], Optional[int]]:
276
+ """Extract and convert token counts safely."""
277
+ input_tokens = None
278
+ output_tokens = None
279
+
280
+ if isinstance(computer_use_usage, dict):
281
+ input_tokens_val = computer_use_usage.get("input_tokens")
282
+ output_tokens_val = computer_use_usage.get("output_tokens")
283
+
284
+ input_tokens = StandardBuiltInToolCostTracking._safe_convert_to_int(
285
+ input_tokens_val
286
+ )
287
+ output_tokens = StandardBuiltInToolCostTracking._safe_convert_to_int(
288
+ output_tokens_val
289
+ )
290
+
291
+ return input_tokens, output_tokens
292
+
293
+ @staticmethod
294
+ def _safe_convert_to_int(value: Any) -> Optional[int]:
295
+ """Safely convert a value to int."""
296
+ if value is not None:
297
+ try:
298
+ return int(value) # type: ignore
299
+ except (TypeError, ValueError):
300
+ return None
301
+ return None
302
+
303
+ @staticmethod
304
+ def response_object_includes_web_search_call(
305
+ response_object: Any, usage: Optional[Usage] = None
306
+ ) -> bool:
307
+ """
308
+ Check if the response object includes a web search call.
309
+
310
+ This covers:
311
+ - Chat Completion Response (ModelResponse)
312
+ - ResponsesAPIResponse (streaming + non-streaming)
313
+ """
314
+ from litellm.types.utils import PromptTokensDetailsWrapper
315
+
316
+ if isinstance(response_object, ModelResponse):
317
+ # chat completions only include url_citation annotations when a web search call is made
318
+ has_url_citations = (
319
+ StandardBuiltInToolCostTracking.response_includes_annotation_type(
320
+ response_object=response_object, annotation_type="url_citation"
321
+ )
322
+ )
323
+ if has_url_citations:
324
+ return True
325
+ # Fallback: Check usage object for providers that use usage instead of annotations
326
+ # (e.g., Vertex AI Gemini uses usage.prompt_tokens_details.web_search_requests)
327
+ if usage is not None:
328
+ if (
329
+ hasattr(usage, "prompt_tokens_details")
330
+ and usage.prompt_tokens_details is not None
331
+ and isinstance(
332
+ usage.prompt_tokens_details, PromptTokensDetailsWrapper
333
+ )
334
+ and hasattr(usage.prompt_tokens_details, "web_search_requests")
335
+ and usage.prompt_tokens_details.web_search_requests is not None
336
+ ):
337
+ return True
338
+ return False
339
+ elif isinstance(response_object, ResponsesAPIResponse):
340
+ # response api explicitly includes web_search_call in the output
341
+ return StandardBuiltInToolCostTracking.response_includes_output_type(
342
+ response_object=response_object, output_type="web_search_call"
343
+ )
344
+ elif usage is not None:
345
+ if (
346
+ hasattr(usage, "server_tool_use")
347
+ and usage.server_tool_use is not None
348
+ and usage.server_tool_use.web_search_requests is not None
349
+ ):
350
+ return True
351
+ elif (
352
+ hasattr(usage, "prompt_tokens_details")
353
+ and usage.prompt_tokens_details is not None
354
+ and isinstance(usage.prompt_tokens_details, PromptTokensDetailsWrapper)
355
+ and hasattr(usage.prompt_tokens_details, "web_search_requests")
356
+ and usage.prompt_tokens_details.web_search_requests is not None
357
+ ):
358
+ return True
359
+
360
+ return False
361
+
362
+ @staticmethod
363
+ def response_object_includes_file_search_call(
364
+ response_object: Any,
365
+ ) -> bool:
366
+ """
367
+ Check if the response object includes a file search call.
368
+
369
+ This covers:
370
+ - Chat Completion Response (ModelResponse)
371
+ - ResponsesAPIResponse (streaming + non-streaming)
372
+ """
373
+ if isinstance(response_object, ModelResponse):
374
+ # chat completions only include file_citation annotations when a file search call is made
375
+ return StandardBuiltInToolCostTracking.response_includes_annotation_type(
376
+ response_object=response_object, annotation_type="file_citation"
377
+ )
378
+ elif isinstance(response_object, ResponsesAPIResponse):
379
+ # response api explicitly includes file_search_call in the output
380
+ return StandardBuiltInToolCostTracking.response_includes_output_type(
381
+ response_object=response_object, output_type="file_search_call"
382
+ )
383
+ return False
384
+
385
+ @staticmethod
386
+ def response_includes_annotation_type(
387
+ response_object: ModelResponse,
388
+ annotation_type: Literal["url_citation", "file_citation"],
389
+ ) -> bool:
390
+ if isinstance(response_object, ModelResponse):
391
+ for choice in response_object.choices:
392
+ message: Optional[Message] = getattr(choice, "message", None)
393
+ if message is None:
394
+ continue
395
+ if annotations := getattr(message, "annotations", None):
396
+ if len(annotations) > 0:
397
+ for annotation in annotations:
398
+ if annotation.get("type", None) == annotation_type:
399
+ return True
400
+ return False
401
+
402
+ @staticmethod
403
+ def response_includes_output_type(
404
+ response_object: ResponsesAPIResponse,
405
+ output_type: Literal["web_search_call", "file_search_call"],
406
+ ) -> bool:
407
+ """
408
+ Check if the ResponsesAPIResponse includes one of the specified output types.
409
+
410
+ This is used for cost tracking of built-in tools.
411
+
412
+ Args:
413
+ response_object: The ResponsesAPIResponse object to check.
414
+ output_type: The type of output to check for.
415
+
416
+ Returns:
417
+ True if the ResponsesAPIResponse includes one of the specified output types, False otherwise.
418
+ """
419
+ output = response_object.output
420
+ for output_item in output:
421
+ _output_type: Optional[str] = getattr(output_item, "type", None)
422
+ if _output_type == output_type:
423
+ return True
424
+ return False
425
+
426
+ @staticmethod
427
+ def _safe_get_model_info(
428
+ model: str, custom_llm_provider: Optional[str] = None
429
+ ) -> Optional[ModelInfo]:
430
+ try:
431
+ return litellm.get_model_info(
432
+ model=model, custom_llm_provider=custom_llm_provider
433
+ )
434
+ except Exception:
435
+ return None
436
+
437
+ @staticmethod
438
+ def get_cost_for_web_search(
439
+ web_search_options: Optional[WebSearchOptions] = None,
440
+ model_info: Optional[ModelInfo] = None,
441
+ ) -> float:
442
+ """
443
+ If request includes `web_search_options`, calculate the cost of the web search.
444
+ """
445
+ web_search_options = web_search_options or {}
446
+ if model_info is None:
447
+ return 0.0
448
+
449
+ search_context_raw: Any = model_info.get("search_context_cost_per_query", {})
450
+ search_context_pricing: SearchContextCostPerQuery = (
451
+ SearchContextCostPerQuery(**search_context_raw)
452
+ if search_context_raw
453
+ else SearchContextCostPerQuery()
454
+ )
455
+ if web_search_options.get("search_context_size", None) == "low":
456
+ return search_context_pricing.get("search_context_size_low", 0.0)
457
+ elif web_search_options.get("search_context_size", None) == "medium":
458
+ return search_context_pricing.get("search_context_size_medium", 0.0)
459
+ elif web_search_options.get("search_context_size", None) == "high":
460
+ return search_context_pricing.get("search_context_size_high", 0.0)
461
+ return StandardBuiltInToolCostTracking.get_default_cost_for_web_search(
462
+ model_info
463
+ )
464
+
465
+ @staticmethod
466
+ def get_default_cost_for_web_search(
467
+ model_info: Optional[ModelInfo] = None,
468
+ ) -> float:
469
+ """
470
+ If no web search options are provided, use the `search_context_size_medium` pricing.
471
+
472
+ https://platform.openai.com/docs/pricing#web-search
473
+ """
474
+ if model_info is None:
475
+ return 0.0
476
+ search_context_raw: Any = (
477
+ model_info.get("search_context_cost_per_query", {}) or {}
478
+ )
479
+ search_context_pricing: SearchContextCostPerQuery = (
480
+ SearchContextCostPerQuery(**search_context_raw)
481
+ if search_context_raw
482
+ else SearchContextCostPerQuery()
483
+ )
484
+ return search_context_pricing.get("search_context_size_medium", 0.0)
485
+
486
+ @staticmethod
487
+ def get_cost_for_file_search(
488
+ file_search: Optional[FileSearchTool] = None,
489
+ provider: Optional[str] = None,
490
+ model_info: Optional[dict] = None,
491
+ storage_gb: Optional[float] = None,
492
+ days: Optional[float] = None,
493
+ ) -> float:
494
+ """ "
495
+ OpenAI: $2.50/1k calls
496
+ Azure: $0.1 USD per 1 GB/Day (storage-based pricing)
497
+
498
+ Doc: https://platform.openai.com/docs/pricing#built-in-tools
499
+ """
500
+ if file_search is None:
501
+ return 0.0
502
+
503
+ # Check if model-specific pricing is available
504
+ if (
505
+ model_info
506
+ and "file_search_cost_per_gb_per_day" in model_info
507
+ and provider == "azure"
508
+ ):
509
+ if storage_gb and days:
510
+ return storage_gb * days * model_info["file_search_cost_per_gb_per_day"]
511
+ elif model_info and "file_search_cost_per_1k_calls" in model_info:
512
+ return model_info["file_search_cost_per_1k_calls"]
513
+
514
+ # Azure has storage-based pricing for file search
515
+ if provider == "azure":
516
+ from litellm.constants import AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY
517
+
518
+ if storage_gb and days:
519
+ return storage_gb * days * AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY
520
+ # Default to 0 if no storage info provided
521
+ return 0.0
522
+
523
+ # Default to OpenAI pricing (per-call based)
524
+ return OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
525
+
526
+ @staticmethod
527
+ def get_cost_for_vector_store(
528
+ vector_store_usage: Optional[dict] = None,
529
+ provider: Optional[str] = None,
530
+ model_info: Optional[dict] = None,
531
+ ) -> float:
532
+ """
533
+ Calculate cost for vector store usage.
534
+
535
+ Azure charges based on storage size and duration.
536
+ """
537
+ if vector_store_usage is None:
538
+ return 0.0
539
+
540
+ storage_gb = vector_store_usage.get("storage_gb", 0.0)
541
+ days = vector_store_usage.get("days", 0.0)
542
+
543
+ # Check if model-specific pricing is available
544
+ if model_info and "vector_store_cost_per_gb_per_day" in model_info:
545
+ return storage_gb * days * model_info["vector_store_cost_per_gb_per_day"]
546
+
547
+ # Azure has different pricing structure for vector store
548
+ if provider == "azure":
549
+ from litellm.constants import AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY
550
+
551
+ return storage_gb * days * AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY
552
+
553
+ # OpenAI doesn't charge separately for vector store (included in embeddings)
554
+ return 0.0
555
+
556
+ @staticmethod
557
+ def get_cost_for_computer_use(
558
+ input_tokens: Optional[int] = None,
559
+ output_tokens: Optional[int] = None,
560
+ provider: Optional[str] = None,
561
+ model_info: Optional[dict] = None,
562
+ ) -> float:
563
+ """
564
+ Calculate cost for computer use feature.
565
+
566
+ Azure: $0.003 USD per 1K input tokens, $0.012 USD per 1K output tokens
567
+ """
568
+ if provider == "azure" and (input_tokens or output_tokens):
569
+ # Check if model-specific pricing is available
570
+ if model_info:
571
+ input_cost = model_info.get(
572
+ "computer_use_input_cost_per_1k_tokens", 0.0
573
+ )
574
+ output_cost = model_info.get(
575
+ "computer_use_output_cost_per_1k_tokens", 0.0
576
+ )
577
+ if input_cost or output_cost:
578
+ total_cost = 0.0
579
+ if input_tokens:
580
+ total_cost += (input_tokens / 1000.0) * input_cost
581
+ if output_tokens:
582
+ total_cost += (output_tokens / 1000.0) * output_cost
583
+ return total_cost
584
+
585
+ # Azure default pricing
586
+ from litellm.constants import (
587
+ AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS,
588
+ AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS,
589
+ )
590
+
591
+ total_cost = 0.0
592
+ if input_tokens:
593
+ total_cost += (
594
+ input_tokens / 1000.0
595
+ ) * AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS
596
+ if output_tokens:
597
+ total_cost += (
598
+ output_tokens / 1000.0
599
+ ) * AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS
600
+ return total_cost
601
+
602
+ # OpenAI doesn't charge separately for computer use yet
603
+ return 0.0
604
+
605
+ @staticmethod
606
+ def _get_code_interpreter_cost_from_model_map(
607
+ provider: str,
608
+ ) -> Optional[float]:
609
+ """
610
+ Get code interpreter cost per session from model cost map.
611
+ """
612
+ import litellm
613
+
614
+ try:
615
+ container_model = f"{provider}/container"
616
+ model_info = litellm.get_model_info(
617
+ model=container_model, custom_llm_provider=provider
618
+ )
619
+ model_key = (
620
+ model_info.get("key")
621
+ if isinstance(model_info, dict)
622
+ else getattr(model_info, "key", None)
623
+ )
624
+
625
+ if model_key and model_key in litellm.model_cost:
626
+ return litellm.model_cost[model_key].get(
627
+ "code_interpreter_cost_per_session"
628
+ )
629
+
630
+ except Exception:
631
+ pass
632
+
633
+ return None
634
+
635
+ @staticmethod
636
+ def get_cost_for_code_interpreter(
637
+ sessions: Optional[int] = None,
638
+ provider: Optional[str] = None,
639
+ model_info: Optional[dict] = None,
640
+ ) -> float:
641
+ """
642
+ Calculate cost for code interpreter feature.
643
+
644
+ Azure: $0.03 USD per session (from model cost map)
645
+ OpenAI: $0.03 USD per session (from model cost map)
646
+ """
647
+ if sessions is None or sessions == 0:
648
+ return 0.0
649
+
650
+ # Check if model-specific pricing is available
651
+ if model_info and "code_interpreter_cost_per_session" in model_info:
652
+ return sessions * model_info["code_interpreter_cost_per_session"]
653
+
654
+ # Try to get cost from model cost map for any provider
655
+ if provider:
656
+ cost_per_session = StandardBuiltInToolCostTracking._get_code_interpreter_cost_from_model_map(
657
+ provider=provider
658
+ )
659
+ if cost_per_session is not None:
660
+ return sessions * cost_per_session
661
+
662
+ return 0.0
663
+
664
+ @staticmethod
665
+ def chat_completion_response_includes_annotations(
666
+ response_object: ModelResponse,
667
+ ) -> bool:
668
+ for _choice in response_object.choices:
669
+ message = getattr(_choice, "message", None)
670
+ if (
671
+ message is not None
672
+ and hasattr(message, "annotations")
673
+ and message.annotations is not None
674
+ and len(message.annotations) > 0
675
+ ):
676
+ return True
677
+ return False
678
+
679
+ @staticmethod
680
+ def _get_web_search_options(kwargs: Dict) -> Optional[WebSearchOptions]:
681
+ if "web_search_options" in kwargs:
682
+ return WebSearchOptions(**kwargs.get("web_search_options", {}))
683
+
684
+ tools = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
685
+ kwargs=kwargs, tool_type="web_search_preview"
686
+ ) or StandardBuiltInToolCostTracking._get_tools_from_kwargs(
687
+ kwargs=kwargs, tool_type="web_search"
688
+ )
689
+ if tools:
690
+ # Look for web search tool in the tools array
691
+ for tool in tools:
692
+ if isinstance(tool, dict):
693
+ if StandardBuiltInToolCostTracking._is_web_search_tool_call(tool):
694
+ return WebSearchOptions(**tool)
695
+ return None
696
+
697
+ @staticmethod
698
+ def _get_tools_from_kwargs(kwargs: Dict, tool_type: str) -> Optional[List[Dict]]:
699
+ if "tools" in kwargs:
700
+ return kwargs.get("tools", [])
701
+ return None
702
+
703
+ @staticmethod
704
+ def _get_file_search_tool_call(kwargs: Dict) -> Optional[FileSearchTool]:
705
+ tools = StandardBuiltInToolCostTracking._get_tools_from_kwargs(
706
+ kwargs, "file_search"
707
+ )
708
+ if tools:
709
+ for tool in tools:
710
+ if isinstance(tool, dict):
711
+ if StandardBuiltInToolCostTracking._is_file_search_tool_call(tool):
712
+ return FileSearchTool(**tool)
713
+ return None
714
+
715
+ @staticmethod
716
+ def _is_web_search_tool_call(tool: Dict) -> bool:
717
+ if tool.get("type", None) == "web_search_preview":
718
+ return True
719
+ if tool.get("type", None) == "web_search":
720
+ return True
721
+ if "search_context_size" in tool:
722
+ return True
723
+ return False
724
+
725
+ @staticmethod
726
+ def _is_file_search_tool_call(tool: Dict) -> bool:
727
+ if tool.get("type", None) == "file_search":
728
+ return True
729
+ return False
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Any, Optional, Union
2
+
3
+ from litellm.types.utils import (
4
+ PromptTokensDetailsWrapper,
5
+ TranscriptionUsageDurationObject,
6
+ TranscriptionUsageTokensObject,
7
+ Usage,
8
+ )
9
+
10
+
11
+ class TranscriptionUsageObjectTransformation:
12
+ @staticmethod
13
+ def is_transcription_usage_object(
14
+ usage_object: Any,
15
+ ) -> bool:
16
+ return isinstance(usage_object, TranscriptionUsageDurationObject) or isinstance(
17
+ usage_object, TranscriptionUsageTokensObject
18
+ )
19
+
20
+ @staticmethod
21
+ def transform_transcription_usage_object(
22
+ usage_object: Union[
23
+ TranscriptionUsageDurationObject, TranscriptionUsageTokensObject
24
+ ],
25
+ ) -> Optional[Usage]:
26
+ if isinstance(usage_object, TranscriptionUsageDurationObject):
27
+ return None
28
+ elif isinstance(usage_object, TranscriptionUsageTokensObject):
29
+ return Usage(
30
+ prompt_tokens=usage_object.input_tokens,
31
+ completion_tokens=usage_object.output_tokens,
32
+ total_tokens=usage_object.total_tokens,
33
+ prompt_tokens_details=PromptTokensDetailsWrapper(
34
+ text_tokens=usage_object.input_token_details.text_tokens,
35
+ audio_tokens=usage_object.input_token_details.audio_tokens,
36
+ ),
37
+ )
38
+ return None
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_cost_calc/utils.py ADDED
@@ -0,0 +1,1034 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # What is this?
2
+ ## Helper utilities for cost_per_token()
3
+
4
+ from typing import Literal, Optional, Tuple, TypedDict, cast
5
+
6
+ import litellm
7
+ from litellm._logging import verbose_logger
8
+ from litellm.types.utils import (
9
+ CacheCreationTokenDetails,
10
+ CallTypes,
11
+ CompletionTokensDetailsWrapper,
12
+ ImageResponse,
13
+ ModelInfo,
14
+ PassthroughCallTypes,
15
+ PromptTokensDetailsWrapper,
16
+ ServiceTier,
17
+ Usage,
18
+ )
19
+ from litellm.utils import get_model_info
20
+
21
+ # Pre-resolved CallTypes enum values for fast membership checks
22
+ _IMAGE_RESPONSE_CALL_TYPES = frozenset(
23
+ {
24
+ CallTypes.image_generation.value,
25
+ CallTypes.aimage_generation.value,
26
+ PassthroughCallTypes.passthrough_image_generation.value,
27
+ CallTypes.image_edit.value,
28
+ CallTypes.aimage_edit.value,
29
+ }
30
+ )
31
+
32
+
33
+ def _is_above_128k(tokens: float) -> bool:
34
+ if tokens > 128000:
35
+ return True
36
+ return False
37
+
38
+
39
+ def get_billable_input_tokens(usage: Usage) -> int:
40
+ """
41
+ Returns the number of billable input tokens.
42
+ Subtracts cached tokens from prompt tokens if applicable.
43
+ """
44
+ details = _parse_prompt_tokens_details(usage)
45
+ return usage.prompt_tokens - details["cache_hit_tokens"]
46
+
47
+
48
+ def select_cost_metric_for_model(
49
+ model_info: ModelInfo,
50
+ ) -> Literal["cost_per_character", "cost_per_token"]:
51
+ """
52
+ Select 'cost_per_character' if model_info has 'input_cost_per_character'
53
+ Select 'cost_per_token' if model_info has 'input_cost_per_token'
54
+ """
55
+ if model_info.get("input_cost_per_character"):
56
+ return "cost_per_character"
57
+ elif model_info.get("input_cost_per_token"):
58
+ return "cost_per_token"
59
+ else:
60
+ raise ValueError(
61
+ f"Model {model_info['key']} does not have 'input_cost_per_character' or 'input_cost_per_token'"
62
+ )
63
+
64
+
65
+ def _generic_cost_per_character(
66
+ model: str,
67
+ custom_llm_provider: str,
68
+ prompt_characters: float,
69
+ completion_characters: float,
70
+ custom_prompt_cost: Optional[float],
71
+ custom_completion_cost: Optional[float],
72
+ ) -> Tuple[Optional[float], Optional[float]]:
73
+ """
74
+ Calculates cost per character for aspeech/speech calls.
75
+
76
+ Calculates the cost per character for a given model, input messages, and response object.
77
+
78
+ Input:
79
+ - model: str, the model name without provider prefix
80
+ - custom_llm_provider: str, "vertex_ai-*"
81
+ - prompt_characters: float, the number of input characters
82
+ - completion_characters: float, the number of output characters
83
+
84
+ Returns:
85
+ Tuple[Optional[float], Optional[float]] - prompt_cost_in_usd, completion_cost_in_usd.
86
+ - returns None if not able to calculate cost.
87
+
88
+ Raises:
89
+ Exception if 'input_cost_per_character' or 'output_cost_per_character' is missing from model_info
90
+ """
91
+ ## GET MODEL INFO
92
+ model_info = litellm.get_model_info(
93
+ model=model, custom_llm_provider=custom_llm_provider
94
+ )
95
+
96
+ ## CALCULATE INPUT COST
97
+ try:
98
+ if custom_prompt_cost is None:
99
+ assert (
100
+ "input_cost_per_character" in model_info
101
+ and model_info["input_cost_per_character"] is not None
102
+ ), "model info for model={} does not have 'input_cost_per_character'-pricing\nmodel_info={}".format(
103
+ model, model_info
104
+ )
105
+ custom_prompt_cost = model_info["input_cost_per_character"]
106
+
107
+ prompt_cost = prompt_characters * custom_prompt_cost
108
+ except Exception as e:
109
+ verbose_logger.exception(
110
+ "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\nDefaulting to None".format(
111
+ str(e)
112
+ )
113
+ )
114
+
115
+ prompt_cost = None
116
+
117
+ ## CALCULATE OUTPUT COST
118
+ try:
119
+ if custom_completion_cost is None:
120
+ assert (
121
+ "output_cost_per_character" in model_info
122
+ and model_info["output_cost_per_character"] is not None
123
+ ), "model info for model={} does not have 'output_cost_per_character'-pricing\nmodel_info={}".format(
124
+ model, model_info
125
+ )
126
+ custom_completion_cost = model_info["output_cost_per_character"]
127
+ completion_cost = completion_characters * custom_completion_cost
128
+ except Exception as e:
129
+ verbose_logger.exception(
130
+ "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\nDefaulting to None".format(
131
+ str(e)
132
+ )
133
+ )
134
+
135
+ completion_cost = None
136
+
137
+ return prompt_cost, completion_cost
138
+
139
+
140
+ def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> str:
141
+ """
142
+ Get the appropriate cost key based on service tier.
143
+
144
+ Args:
145
+ base_key: The base cost key (e.g., "input_cost_per_token")
146
+ service_tier: The service tier ("flex", "priority", or None for standard)
147
+
148
+ Returns:
149
+ str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token")
150
+ """
151
+ if service_tier is None:
152
+ return base_key
153
+
154
+ # Only use service tier specific keys for "flex" and "priority"
155
+ if service_tier.lower() in [ServiceTier.FLEX.value, ServiceTier.PRIORITY.value]:
156
+ return f"{base_key}_{service_tier.lower()}"
157
+
158
+ # For any other service tier, use standard pricing
159
+ return base_key
160
+
161
+
162
+ def _get_token_base_cost(
163
+ model_info: ModelInfo, usage: Usage, service_tier: Optional[str] = None
164
+ ) -> Tuple[float, float, float, float, float]:
165
+ """
166
+ Return prompt cost, completion cost, and cache costs for a given model and usage.
167
+
168
+ If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set,
169
+ then we use the corresponding threshold cost for all token types.
170
+
171
+ Returns:
172
+ Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
173
+ """
174
+ # Get service tier aware cost keys
175
+ input_cost_key = _get_service_tier_cost_key("input_cost_per_token", service_tier)
176
+ output_cost_key = _get_service_tier_cost_key("output_cost_per_token", service_tier)
177
+ cache_creation_cost_key = _get_service_tier_cost_key(
178
+ "cache_creation_input_token_cost", service_tier
179
+ )
180
+ cache_read_cost_key = _get_service_tier_cost_key(
181
+ "cache_read_input_token_cost", service_tier
182
+ )
183
+
184
+ prompt_base_cost = cast(float, _get_cost_per_unit(model_info, input_cost_key))
185
+ completion_base_cost = cast(float, _get_cost_per_unit(model_info, output_cost_key))
186
+
187
+ # For image generation models that don't have output_cost_per_token,
188
+ # use output_cost_per_image_token as the base cost (all output tokens are image tokens)
189
+ if completion_base_cost == 0.0 or completion_base_cost is None:
190
+ output_image_cost = _get_cost_per_unit(
191
+ model_info, "output_cost_per_image_token", None
192
+ )
193
+ if output_image_cost is not None:
194
+ completion_base_cost = cast(float, output_image_cost)
195
+ cache_creation_cost = cast(
196
+ float, _get_cost_per_unit(model_info, cache_creation_cost_key)
197
+ )
198
+ cache_creation_cost_above_1hr = cast(
199
+ float,
200
+ _get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),
201
+ )
202
+ cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key))
203
+
204
+ ## CHECK IF ABOVE THRESHOLD
205
+ # Optimization: collect threshold keys first to avoid sorting all model_info keys.
206
+ # Most models don't have threshold pricing, so we can return early.
207
+ # Exclude service_tier-specific variants (e.g. input_cost_per_token_above_200k_tokens_priority)
208
+ # so that the threshold detection loop only processes standard keys. The
209
+ # service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key.
210
+ threshold_keys = [
211
+ k
212
+ for k in model_info
213
+ if k.startswith("input_cost_per_token_above_")
214
+ and not any(k.endswith(f"_{st.value}") for st in ServiceTier)
215
+ ]
216
+ if not threshold_keys:
217
+ return (
218
+ prompt_base_cost,
219
+ completion_base_cost,
220
+ cache_creation_cost,
221
+ cache_creation_cost_above_1hr,
222
+ cache_read_cost,
223
+ )
224
+
225
+ # Only sort the threshold keys (typically 1-2 keys instead of 66+)
226
+ threshold: Optional[float] = None
227
+ for key in sorted(threshold_keys, reverse=True):
228
+ value = model_info.get(key)
229
+ if value is not None:
230
+ try:
231
+ # Handle both formats: _above_128k_tokens and _above_128_tokens
232
+ threshold_str = key.split("_above_")[1].split("_tokens")[0]
233
+ threshold = float(threshold_str.replace("k", "")) * (
234
+ 1000 if "k" in threshold_str else 1
235
+ )
236
+ if usage.prompt_tokens > threshold:
237
+ # Prefer a service_tier-specific above-threshold key when available,
238
+ # e.g. input_cost_per_token_priority_above_200k_tokens for Gemini
239
+ # ON_DEMAND_PRIORITY. Falls back to the standard key automatically
240
+ # via _get_cost_per_unit's service_tier fallback logic.
241
+ tiered_input_key = (
242
+ _get_service_tier_cost_key(
243
+ f"input_cost_per_token_above_{threshold_str}_tokens",
244
+ service_tier,
245
+ )
246
+ if service_tier
247
+ else key
248
+ )
249
+ prompt_base_cost = cast(
250
+ float,
251
+ _get_cost_per_unit(
252
+ model_info, tiered_input_key, prompt_base_cost
253
+ ),
254
+ )
255
+ tiered_output_key = (
256
+ _get_service_tier_cost_key(
257
+ f"output_cost_per_token_above_{threshold_str}_tokens",
258
+ service_tier,
259
+ )
260
+ if service_tier
261
+ else f"output_cost_per_token_above_{threshold_str}_tokens"
262
+ )
263
+ completion_base_cost = cast(
264
+ float,
265
+ _get_cost_per_unit(
266
+ model_info,
267
+ tiered_output_key,
268
+ completion_base_cost,
269
+ ),
270
+ )
271
+
272
+ # Apply tiered pricing to cache costs
273
+ cache_creation_tiered_key = (
274
+ f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
275
+ )
276
+ cache_creation_1hr_tiered_key = f"cache_creation_input_token_cost_above_1hr_above_{threshold_str}_tokens"
277
+ cache_read_tiered_key = (
278
+ f"cache_read_input_token_cost_above_{threshold_str}_tokens"
279
+ )
280
+
281
+ if cache_creation_tiered_key in model_info:
282
+ cache_creation_cost = cast(
283
+ float,
284
+ _get_cost_per_unit(
285
+ model_info,
286
+ cache_creation_tiered_key,
287
+ cache_creation_cost,
288
+ ),
289
+ )
290
+
291
+ if cache_creation_1hr_tiered_key in model_info:
292
+ cache_creation_cost_above_1hr = cast(
293
+ float,
294
+ _get_cost_per_unit(
295
+ model_info,
296
+ cache_creation_1hr_tiered_key,
297
+ cache_creation_cost_above_1hr,
298
+ ),
299
+ )
300
+
301
+ if cache_read_tiered_key in model_info:
302
+ cache_read_cost = cast(
303
+ float,
304
+ _get_cost_per_unit(
305
+ model_info, cache_read_tiered_key, cache_read_cost
306
+ ),
307
+ )
308
+
309
+ break
310
+ except (IndexError, ValueError):
311
+ continue
312
+ except Exception:
313
+ continue
314
+
315
+ return (
316
+ prompt_base_cost,
317
+ completion_base_cost,
318
+ cache_creation_cost,
319
+ cache_creation_cost_above_1hr,
320
+ cache_read_cost,
321
+ )
322
+
323
+
324
+ def calculate_cost_component(
325
+ model_info: ModelInfo, cost_key: str, usage_value: Optional[float]
326
+ ) -> float:
327
+ """
328
+ Generic cost calculator for any usage component
329
+
330
+ Args:
331
+ model_info: Dictionary containing cost information
332
+ cost_key: The key for the cost multiplier in model_info (e.g., 'input_cost_per_audio_token')
333
+ usage_value: The actual usage value (e.g., number of tokens, characters, seconds)
334
+
335
+ Returns:
336
+ float: The calculated cost
337
+ """
338
+ cost_per_unit = _get_cost_per_unit(model_info, cost_key)
339
+ if (
340
+ cost_per_unit is not None
341
+ and isinstance(cost_per_unit, float)
342
+ and usage_value is not None
343
+ and usage_value > 0
344
+ ):
345
+ return float(usage_value) * cost_per_unit
346
+ return 0.0
347
+
348
+
349
+ def _get_cost_per_unit(
350
+ model_info: ModelInfo, cost_key: str, default_value: Optional[float] = 0.0
351
+ ) -> Optional[float]:
352
+ # Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
353
+ cost_per_unit = model_info.get(cost_key)
354
+ if isinstance(cost_per_unit, float):
355
+ return cost_per_unit
356
+ if isinstance(cost_per_unit, int):
357
+ return float(cost_per_unit)
358
+ if isinstance(cost_per_unit, str):
359
+ try:
360
+ return float(cost_per_unit)
361
+ except ValueError:
362
+ verbose_logger.exception(
363
+ f"litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - {cost_per_unit}\nDefaulting to 0.0"
364
+ )
365
+
366
+ # If the service tier key doesn't exist or is None, try to fall back to the standard key
367
+ if cost_per_unit is None:
368
+ # Check if any service tier suffix exists in the cost key using ServiceTier enum
369
+ for service_tier in ServiceTier:
370
+ suffix = f"_{service_tier.value}"
371
+ if suffix in cost_key:
372
+ # Extract the base key by removing the matched suffix
373
+ base_key = cost_key.replace(suffix, "")
374
+ fallback_cost = model_info.get(base_key)
375
+ if isinstance(fallback_cost, float):
376
+ return fallback_cost
377
+ if isinstance(fallback_cost, int):
378
+ return float(fallback_cost)
379
+ if isinstance(fallback_cost, str):
380
+ try:
381
+ return float(fallback_cost)
382
+ except ValueError:
383
+ verbose_logger.exception(
384
+ f"litellm.litellm_core_utils.llm_cost_calc.utils.py::_get_cost_per_unit(): Exception occured - {fallback_cost}\nDefaulting to 0.0"
385
+ )
386
+ break # Only try the first matching suffix
387
+
388
+ return default_value
389
+
390
+
391
+ def calculate_cache_writing_cost(
392
+ cache_creation_tokens: int,
393
+ cache_creation_token_details: Optional[CacheCreationTokenDetails],
394
+ cache_creation_cost_above_1hr: float,
395
+ cache_creation_cost: float,
396
+ ) -> float:
397
+ """
398
+ Adjust cost of cache creation tokens based on the cache creation token details.
399
+ """
400
+ total_cost: float = 0.0
401
+ if cache_creation_token_details is not None:
402
+ # get the number of 5m and 1h cache creation tokens
403
+ cache_creation_tokens_5m = (
404
+ cache_creation_token_details.ephemeral_5m_input_tokens
405
+ )
406
+ cache_creation_tokens_1h = (
407
+ cache_creation_token_details.ephemeral_1h_input_tokens
408
+ )
409
+ # add the number of 5m and 1h cache creation tokens to the cache creation tokens
410
+ total_cost += (
411
+ cache_creation_tokens_5m * cache_creation_cost
412
+ if cache_creation_tokens_5m is not None
413
+ else 0.0
414
+ )
415
+ total_cost += (
416
+ cache_creation_tokens_1h * cache_creation_cost_above_1hr
417
+ if cache_creation_tokens_1h is not None
418
+ else 0.0
419
+ )
420
+ else:
421
+ total_cost += cache_creation_tokens * cache_creation_cost
422
+ return total_cost
423
+
424
+
425
+ class PromptTokensDetailsResult(TypedDict):
426
+ cache_hit_tokens: int
427
+ cache_creation_tokens: int
428
+ cache_creation_token_details: Optional[CacheCreationTokenDetails]
429
+ text_tokens: int
430
+ audio_tokens: int
431
+ image_tokens: int
432
+ character_count: int
433
+ image_count: int
434
+ video_length_seconds: float
435
+
436
+
437
+ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
438
+ cache_hit_tokens = (
439
+ cast(Optional[int], getattr(usage.prompt_tokens_details, "cached_tokens", 0))
440
+ or 0
441
+ )
442
+ cache_creation_tokens = (
443
+ cast(
444
+ Optional[int],
445
+ getattr(usage.prompt_tokens_details, "cache_creation_tokens", 0),
446
+ )
447
+ or 0
448
+ )
449
+ cache_creation_token_details = (
450
+ cast(
451
+ Optional[CacheCreationTokenDetails],
452
+ getattr(usage.prompt_tokens_details, "cache_creation_token_details", None),
453
+ )
454
+ or None
455
+ )
456
+ text_tokens = (
457
+ cast(Optional[int], getattr(usage.prompt_tokens_details, "text_tokens", None))
458
+ or 0 # default to prompt tokens, if this field is not set
459
+ )
460
+ audio_tokens = (
461
+ cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
462
+ or 0
463
+ )
464
+ image_tokens = (
465
+ cast(Optional[int], getattr(usage.prompt_tokens_details, "image_tokens", 0))
466
+ or 0
467
+ )
468
+ character_count = (
469
+ cast(
470
+ Optional[int],
471
+ getattr(usage.prompt_tokens_details, "character_count", 0),
472
+ )
473
+ or 0
474
+ )
475
+ image_count = (
476
+ cast(Optional[int], getattr(usage.prompt_tokens_details, "image_count", 0)) or 0
477
+ )
478
+ video_length_seconds = (
479
+ cast(
480
+ Optional[float],
481
+ getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
482
+ )
483
+ or 0.0
484
+ )
485
+
486
+ return PromptTokensDetailsResult(
487
+ cache_hit_tokens=cache_hit_tokens,
488
+ cache_creation_tokens=cache_creation_tokens,
489
+ cache_creation_token_details=cache_creation_token_details,
490
+ text_tokens=text_tokens,
491
+ audio_tokens=audio_tokens,
492
+ image_tokens=image_tokens,
493
+ character_count=character_count,
494
+ image_count=image_count,
495
+ video_length_seconds=float(video_length_seconds),
496
+ )
497
+
498
+
499
+ class CompletionTokensDetailsResult(TypedDict):
500
+ audio_tokens: int
501
+ text_tokens: int
502
+ reasoning_tokens: int
503
+ image_tokens: int
504
+
505
+
506
+ def _parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResult:
507
+ audio_tokens = (
508
+ cast(
509
+ Optional[int],
510
+ getattr(usage.completion_tokens_details, "audio_tokens", 0),
511
+ )
512
+ or 0
513
+ )
514
+ text_tokens = (
515
+ cast(
516
+ Optional[int],
517
+ getattr(usage.completion_tokens_details, "text_tokens", None),
518
+ )
519
+ or 0 # default to completion tokens, if this field is not set
520
+ )
521
+ reasoning_tokens = (
522
+ cast(
523
+ Optional[int],
524
+ getattr(usage.completion_tokens_details, "reasoning_tokens", 0),
525
+ )
526
+ or 0
527
+ )
528
+ image_tokens = (
529
+ cast(
530
+ Optional[int],
531
+ getattr(usage.completion_tokens_details, "image_tokens", 0),
532
+ )
533
+ or 0
534
+ )
535
+
536
+ return CompletionTokensDetailsResult(
537
+ audio_tokens=audio_tokens,
538
+ text_tokens=text_tokens,
539
+ reasoning_tokens=reasoning_tokens,
540
+ image_tokens=image_tokens,
541
+ )
542
+
543
+
544
+ def _calculate_input_cost(
545
+ prompt_tokens_details: PromptTokensDetailsResult,
546
+ model_info: ModelInfo,
547
+ prompt_base_cost: float,
548
+ cache_read_cost: float,
549
+ cache_creation_cost: float,
550
+ cache_creation_cost_above_1hr: float,
551
+ service_tier: Optional[str] = None,
552
+ ) -> float:
553
+ """
554
+ Calculates the input cost for a given model, prompt tokens, and completion tokens.
555
+ """
556
+ prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
557
+
558
+ ### CACHE READ COST - Now uses tiered pricing
559
+ prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
560
+
561
+ ### AUDIO COST
562
+ if prompt_tokens_details["audio_tokens"]:
563
+ audio_cost_key = _get_service_tier_cost_key(
564
+ "input_cost_per_audio_token", service_tier
565
+ )
566
+ prompt_cost += calculate_cost_component(
567
+ model_info, audio_cost_key, prompt_tokens_details["audio_tokens"]
568
+ )
569
+
570
+ ### IMAGE TOKEN COST
571
+ if prompt_tokens_details["image_tokens"]:
572
+ # For image token costs:
573
+ # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
574
+ image_token_cost_key = "input_cost_per_image_token"
575
+ if model_info.get(image_token_cost_key) is None:
576
+ image_token_cost_key = "input_cost_per_token"
577
+ prompt_cost += calculate_cost_component(
578
+ model_info, image_token_cost_key, prompt_tokens_details["image_tokens"]
579
+ )
580
+
581
+ ### CACHE WRITING COST - Now uses tiered pricing
582
+ if (
583
+ prompt_tokens_details["cache_creation_tokens"]
584
+ or prompt_tokens_details["cache_creation_token_details"] is not None
585
+ ):
586
+ prompt_cost += calculate_cache_writing_cost(
587
+ cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
588
+ cache_creation_token_details=prompt_tokens_details[
589
+ "cache_creation_token_details"
590
+ ],
591
+ cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
592
+ cache_creation_cost=cache_creation_cost,
593
+ )
594
+
595
+ ### CHARACTER COST
596
+ if prompt_tokens_details["character_count"]:
597
+ prompt_cost += calculate_cost_component(
598
+ model_info,
599
+ "input_cost_per_character",
600
+ prompt_tokens_details["character_count"],
601
+ )
602
+
603
+ ### IMAGE COUNT COST
604
+ if prompt_tokens_details["image_count"]:
605
+ prompt_cost += calculate_cost_component(
606
+ model_info, "input_cost_per_image", prompt_tokens_details["image_count"]
607
+ )
608
+
609
+ ### VIDEO LENGTH COST
610
+ if prompt_tokens_details["video_length_seconds"]:
611
+ prompt_cost += calculate_cost_component(
612
+ model_info,
613
+ "input_cost_per_video_per_second",
614
+ prompt_tokens_details["video_length_seconds"],
615
+ )
616
+
617
+ return prompt_cost
618
+
619
+
620
+ def generic_cost_per_token( # noqa: PLR0915
621
+ model: str,
622
+ usage: Usage,
623
+ custom_llm_provider: str,
624
+ service_tier: Optional[str] = None,
625
+ ) -> Tuple[float, float]:
626
+ """
627
+ Calculates the cost per token for a given model, prompt tokens, and completion tokens.
628
+
629
+ Handles context caching as well.
630
+
631
+ Input:
632
+ - model: str, the model name without provider prefix
633
+ - usage: LiteLLM Usage block, containing anthropic caching information
634
+
635
+ Returns:
636
+ Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
637
+ """
638
+
639
+ ## GET MODEL INFO
640
+ model_info = get_model_info(model=model, custom_llm_provider=custom_llm_provider)
641
+
642
+ ## CALCULATE INPUT COST
643
+ ### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing)
644
+ prompt_cost = 0.0
645
+ ### PROCESSING COST
646
+ prompt_tokens_details = PromptTokensDetailsResult(
647
+ cache_hit_tokens=0,
648
+ cache_creation_tokens=0,
649
+ cache_creation_token_details=None,
650
+ text_tokens=usage.prompt_tokens,
651
+ audio_tokens=0,
652
+ image_tokens=0,
653
+ character_count=0,
654
+ image_count=0,
655
+ video_length_seconds=0.0,
656
+ )
657
+ if usage.prompt_tokens_details:
658
+ prompt_tokens_details = _parse_prompt_tokens_details(usage)
659
+
660
+ ## EDGE CASE - text tokens not set or includes cached tokens (double-counting)
661
+ ## Some providers (like xAI) report text_tokens = prompt_tokens (including cached)
662
+ ## We detect this when: text_tokens + cached_tokens + other > prompt_tokens
663
+ ## Ref: https://github.com/BerriAI/litellm/issues/19680, #14874, #14875
664
+
665
+ cache_hit = prompt_tokens_details["cache_hit_tokens"]
666
+ text_tokens = prompt_tokens_details["text_tokens"]
667
+ audio_tokens = prompt_tokens_details["audio_tokens"]
668
+ cache_creation = prompt_tokens_details["cache_creation_tokens"]
669
+ image_tokens = prompt_tokens_details["image_tokens"]
670
+
671
+ # Check for double-counting: sum of details > prompt_tokens means overlap
672
+ total_details = (
673
+ text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens
674
+ )
675
+ has_double_counting = cache_hit > 0 and total_details > usage.prompt_tokens
676
+
677
+ if (
678
+ text_tokens == 0 and prompt_tokens_details["image_count"] == 0
679
+ ) or has_double_counting:
680
+ text_tokens = (
681
+ usage.prompt_tokens
682
+ - cache_hit
683
+ - audio_tokens
684
+ - cache_creation
685
+ - image_tokens
686
+ )
687
+ prompt_tokens_details["text_tokens"] = text_tokens
688
+
689
+ (
690
+ prompt_base_cost,
691
+ completion_base_cost,
692
+ cache_creation_cost,
693
+ cache_creation_cost_above_1hr,
694
+ cache_read_cost,
695
+ ) = _get_token_base_cost(
696
+ model_info=model_info, usage=usage, service_tier=service_tier
697
+ )
698
+
699
+ prompt_cost = _calculate_input_cost(
700
+ prompt_tokens_details=prompt_tokens_details,
701
+ model_info=model_info,
702
+ prompt_base_cost=prompt_base_cost,
703
+ cache_read_cost=cache_read_cost,
704
+ cache_creation_cost=cache_creation_cost,
705
+ cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
706
+ service_tier=service_tier,
707
+ )
708
+
709
+ ## CALCULATE OUTPUT COST
710
+ text_tokens = 0
711
+ audio_tokens = 0
712
+ reasoning_tokens = 0
713
+ image_tokens = 0
714
+ is_text_tokens_total = False
715
+ if usage.completion_tokens_details is not None:
716
+ completion_tokens_details = _parse_completion_tokens_details(usage)
717
+ audio_tokens = completion_tokens_details["audio_tokens"]
718
+ text_tokens = completion_tokens_details["text_tokens"]
719
+ reasoning_tokens = completion_tokens_details["reasoning_tokens"]
720
+ image_tokens = completion_tokens_details["image_tokens"]
721
+
722
+ # Handle text_tokens calculation:
723
+ # 1. If text_tokens is explicitly provided and > 0, use it
724
+ # 2. If there's a breakdown (reasoning/audio/image tokens), calculate text_tokens as the remainder
725
+ # 3. If no breakdown at all, assume all completion_tokens are text_tokens
726
+ has_token_breakdown = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0
727
+ if text_tokens == 0:
728
+ if has_token_breakdown:
729
+ # Calculate text tokens as remainder when we have a breakdown
730
+ # This handles cases like OpenAI's reasoning models where text_tokens isn't provided
731
+ text_tokens = max(
732
+ 0,
733
+ usage.completion_tokens
734
+ - reasoning_tokens
735
+ - audio_tokens
736
+ - image_tokens,
737
+ )
738
+ else:
739
+ # No breakdown at all, all tokens are text tokens
740
+ text_tokens = usage.completion_tokens
741
+ is_text_tokens_total = True
742
+ ## TEXT COST
743
+ completion_cost = float(text_tokens) * completion_base_cost
744
+
745
+ ## AUDIO COST
746
+ if not is_text_tokens_total and audio_tokens is not None and audio_tokens > 0:
747
+ _output_cost_per_audio_token = _get_cost_per_unit(
748
+ model_info, "output_cost_per_audio_token", None
749
+ )
750
+ _output_cost_per_audio_token = (
751
+ _output_cost_per_audio_token
752
+ if _output_cost_per_audio_token is not None
753
+ else completion_base_cost
754
+ )
755
+ completion_cost += float(audio_tokens) * _output_cost_per_audio_token
756
+
757
+ ## REASONING COST
758
+ if not is_text_tokens_total and reasoning_tokens and reasoning_tokens > 0:
759
+ _output_cost_per_reasoning_token = _get_cost_per_unit(
760
+ model_info, "output_cost_per_reasoning_token", None
761
+ )
762
+ _output_cost_per_reasoning_token = (
763
+ _output_cost_per_reasoning_token
764
+ if _output_cost_per_reasoning_token is not None
765
+ else completion_base_cost
766
+ )
767
+ completion_cost += float(reasoning_tokens) * _output_cost_per_reasoning_token
768
+
769
+ ## IMAGE COST
770
+ if not is_text_tokens_total and image_tokens and image_tokens > 0:
771
+ _output_cost_per_image_token = _get_cost_per_unit(
772
+ model_info, "output_cost_per_image_token", None
773
+ )
774
+ _output_cost_per_image_token = (
775
+ _output_cost_per_image_token
776
+ if _output_cost_per_image_token is not None
777
+ else completion_base_cost
778
+ )
779
+ completion_cost += float(image_tokens) * _output_cost_per_image_token
780
+
781
+ return prompt_cost, completion_cost
782
+
783
+
784
+ def calculate_image_response_cost_from_usage(
785
+ model: str,
786
+ image_response: ImageResponse,
787
+ custom_llm_provider: str,
788
+ ) -> Optional[float]:
789
+ """
790
+ Calculate image generation cost from usage metadata when available.
791
+
792
+ Returns:
793
+ Optional[float]: total cost from token usage, or None when usage metadata
794
+ is missing/incomplete and caller should fall back to flat per-image pricing.
795
+ """
796
+ usage = image_response.usage
797
+ if usage is None:
798
+ return None
799
+
800
+ prompt_tokens = usage.input_tokens
801
+ completion_tokens = usage.output_tokens
802
+ total_tokens = usage.total_tokens
803
+
804
+ if prompt_tokens is None or completion_tokens is None or total_tokens is None:
805
+ return None
806
+
807
+ # ImageResponse may carry a default zeroed usage object even when provider
808
+ # usage metadata is absent. Treat this as missing usage and fall back.
809
+ if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0:
810
+ return None
811
+
812
+ input_tokens_details = getattr(usage, "input_tokens_details", None)
813
+ prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
814
+ if input_tokens_details is not None:
815
+ prompt_tokens_details = PromptTokensDetailsWrapper(
816
+ text_tokens=getattr(input_tokens_details, "text_tokens", None),
817
+ image_tokens=getattr(input_tokens_details, "image_tokens", None),
818
+ cached_tokens=0,
819
+ )
820
+
821
+ normalized_usage = Usage(
822
+ prompt_tokens=prompt_tokens,
823
+ completion_tokens=completion_tokens,
824
+ total_tokens=total_tokens,
825
+ prompt_tokens_details=prompt_tokens_details,
826
+ completion_tokens_details=CompletionTokensDetailsWrapper(
827
+ text_tokens=0,
828
+ image_tokens=completion_tokens,
829
+ reasoning_tokens=0,
830
+ audio_tokens=0,
831
+ ),
832
+ )
833
+
834
+ prompt_cost, completion_cost = generic_cost_per_token(
835
+ model=model,
836
+ usage=normalized_usage,
837
+ custom_llm_provider=custom_llm_provider,
838
+ )
839
+ return prompt_cost + completion_cost
840
+
841
+
842
+ class CostCalculatorUtils:
843
+ @staticmethod
844
+ def _call_type_has_image_response(call_type: str) -> bool:
845
+ """
846
+ Returns True if the call type has an image response
847
+
848
+ eg calls that have image response:
849
+ - Image Generation
850
+ - Image Edit
851
+ - Passthrough Image Generation
852
+ """
853
+ return call_type in _IMAGE_RESPONSE_CALL_TYPES
854
+
855
+ @staticmethod
856
+ def route_image_generation_cost_calculator(
857
+ model: str,
858
+ completion_response: ImageResponse,
859
+ custom_llm_provider: Optional[str] = None,
860
+ quality: Optional[str] = None,
861
+ n: Optional[int] = None,
862
+ size: Optional[str] = None,
863
+ optional_params: Optional[dict] = None,
864
+ call_type: Optional[str] = None,
865
+ ) -> float:
866
+ """
867
+ Route the image generation cost calculator based on the custom_llm_provider
868
+ """
869
+ from litellm.cost_calculator import default_image_cost_calculator
870
+ from litellm.llms.azure_ai.image_generation.cost_calculator import (
871
+ cost_calculator as azure_ai_image_cost_calculator,
872
+ )
873
+ from litellm.llms.bedrock.image_generation.cost_calculator import (
874
+ cost_calculator as bedrock_image_cost_calculator,
875
+ )
876
+ from litellm.llms.gemini.image_generation.cost_calculator import (
877
+ cost_calculator as gemini_image_cost_calculator,
878
+ )
879
+ from litellm.llms.recraft.cost_calculator import (
880
+ cost_calculator as recraft_image_cost_calculator,
881
+ )
882
+ from litellm.llms.vertex_ai.image_generation.cost_calculator import (
883
+ cost_calculator as vertex_ai_image_cost_calculator,
884
+ )
885
+
886
+ if size is None:
887
+ size = completion_response.size or "1024-x-1024"
888
+ if quality is None:
889
+ quality = completion_response.quality or "standard"
890
+ if n is None:
891
+ n = len(completion_response.data) if completion_response.data else 0
892
+
893
+ if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
894
+ if isinstance(completion_response, ImageResponse):
895
+ return vertex_ai_image_cost_calculator(
896
+ model=model,
897
+ image_response=completion_response,
898
+ )
899
+ elif custom_llm_provider == litellm.LlmProviders.BEDROCK.value:
900
+ if isinstance(completion_response, ImageResponse):
901
+ return bedrock_image_cost_calculator(
902
+ model=model,
903
+ size=size,
904
+ image_response=completion_response,
905
+ optional_params=optional_params,
906
+ )
907
+ raise TypeError(
908
+ "completion_response must be of type ImageResponse for bedrock image cost calculation"
909
+ )
910
+ elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
911
+ from litellm.llms.recraft.cost_calculator import (
912
+ cost_calculator as recraft_image_cost_calculator,
913
+ )
914
+
915
+ return recraft_image_cost_calculator(
916
+ model=model,
917
+ image_response=completion_response,
918
+ )
919
+ elif custom_llm_provider == litellm.LlmProviders.AIML.value:
920
+ from litellm.llms.aiml.image_generation.cost_calculator import (
921
+ cost_calculator as aiml_image_cost_calculator,
922
+ )
923
+
924
+ return aiml_image_cost_calculator(
925
+ model=model,
926
+ image_response=completion_response,
927
+ )
928
+ elif custom_llm_provider == litellm.LlmProviders.COMETAPI.value:
929
+ from litellm.llms.cometapi.image_generation.cost_calculator import (
930
+ cost_calculator as cometapi_image_cost_calculator,
931
+ )
932
+
933
+ return cometapi_image_cost_calculator(
934
+ model=model,
935
+ image_response=completion_response,
936
+ )
937
+ elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
938
+ if call_type in (
939
+ CallTypes.image_edit.value,
940
+ CallTypes.aimage_edit.value,
941
+ ):
942
+ from litellm.llms.gemini.image_edit.cost_calculator import (
943
+ cost_calculator as gemini_image_edit_cost_calculator,
944
+ )
945
+
946
+ return gemini_image_edit_cost_calculator(
947
+ model=model,
948
+ image_response=completion_response,
949
+ )
950
+ from litellm.llms.gemini.image_generation.cost_calculator import (
951
+ cost_calculator as gemini_image_cost_calculator,
952
+ )
953
+
954
+ return gemini_image_cost_calculator(
955
+ model=model,
956
+ image_response=completion_response,
957
+ )
958
+ elif custom_llm_provider == litellm.LlmProviders.AZURE_AI.value:
959
+ return azure_ai_image_cost_calculator(
960
+ model=model,
961
+ image_response=completion_response,
962
+ )
963
+ elif custom_llm_provider == litellm.LlmProviders.FAL_AI.value:
964
+ from litellm.llms.fal_ai.cost_calculator import (
965
+ cost_calculator as fal_ai_image_cost_calculator,
966
+ )
967
+
968
+ return fal_ai_image_cost_calculator(
969
+ model=model,
970
+ image_response=completion_response,
971
+ )
972
+ elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
973
+ from litellm.llms.runwayml.cost_calculator import (
974
+ cost_calculator as runwayml_image_cost_calculator,
975
+ )
976
+
977
+ return runwayml_image_cost_calculator(
978
+ model=model,
979
+ image_response=completion_response,
980
+ )
981
+ elif custom_llm_provider == litellm.LlmProviders.OPENAI.value:
982
+ # Check if this is a gpt-image model (token-based pricing)
983
+ model_lower = model.lower()
984
+ if "gpt-image-1" in model_lower:
985
+ from litellm.llms.openai.image_generation.cost_calculator import (
986
+ cost_calculator as openai_gpt_image_cost_calculator,
987
+ )
988
+
989
+ return openai_gpt_image_cost_calculator(
990
+ model=model,
991
+ image_response=completion_response,
992
+ custom_llm_provider=custom_llm_provider,
993
+ )
994
+ # Fall through to default for DALL-E models
995
+ return default_image_cost_calculator(
996
+ model=model,
997
+ quality=quality,
998
+ custom_llm_provider=custom_llm_provider,
999
+ n=n,
1000
+ size=size,
1001
+ optional_params=optional_params,
1002
+ )
1003
+ elif custom_llm_provider == litellm.LlmProviders.AZURE.value:
1004
+ # Check if this is a gpt-image model (token-based pricing)
1005
+ model_lower = model.lower()
1006
+ if "gpt-image-1" in model_lower:
1007
+ from litellm.llms.openai.image_generation.cost_calculator import (
1008
+ cost_calculator as openai_gpt_image_cost_calculator,
1009
+ )
1010
+
1011
+ return openai_gpt_image_cost_calculator(
1012
+ model=model,
1013
+ image_response=completion_response,
1014
+ custom_llm_provider=custom_llm_provider,
1015
+ )
1016
+ # Fall through to default for DALL-E models
1017
+ return default_image_cost_calculator(
1018
+ model=model,
1019
+ quality=quality,
1020
+ custom_llm_provider=custom_llm_provider,
1021
+ n=n,
1022
+ size=size,
1023
+ optional_params=optional_params,
1024
+ )
1025
+ else:
1026
+ return default_image_cost_calculator(
1027
+ model=model,
1028
+ quality=quality,
1029
+ custom_llm_provider=custom_llm_provider,
1030
+ n=n,
1031
+ size=size,
1032
+ optional_params=optional_params,
1033
+ )
1034
+ return 0.0
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_request_utils.py ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Dict, Optional
2
+
3
+ import litellm
4
+
5
+
6
+ def _ensure_extra_body_is_safe(extra_body: Optional[Dict]) -> Optional[Dict]:
7
+ """
8
+ Ensure that the extra_body sent in the request is safe, otherwise users will see this error
9
+
10
+ "Object of type TextPromptClient is not JSON serializable
11
+
12
+
13
+ Relevant Issue: https://github.com/BerriAI/litellm/issues/4140
14
+ """
15
+ if extra_body is None:
16
+ return None
17
+
18
+ if not isinstance(extra_body, dict):
19
+ return extra_body
20
+
21
+ if "metadata" in extra_body and isinstance(extra_body["metadata"], dict):
22
+ if "prompt" in extra_body["metadata"]:
23
+ _prompt = extra_body["metadata"].get("prompt")
24
+
25
+ # users can send Langfuse TextPromptClient objects, so we need to convert them to dicts
26
+ # Langfuse TextPromptClients have .__dict__ attribute
27
+ if _prompt is not None and hasattr(_prompt, "__dict__"):
28
+ extra_body["metadata"]["prompt"] = _prompt.__dict__
29
+
30
+ return extra_body
31
+
32
+
33
+ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
34
+ """
35
+ Pick the n cheapest chat models from the LLM provider.
36
+
37
+ Args:
38
+ custom_llm_provider (str): The name of the LLM provider.
39
+ n (int): The number of cheapest models to return.
40
+
41
+ Returns:
42
+ list[str]: A list of the n cheapest chat models.
43
+ """
44
+ if custom_llm_provider not in litellm.models_by_provider:
45
+ return []
46
+
47
+ known_models = litellm.models_by_provider.get(custom_llm_provider, [])
48
+ model_costs = []
49
+
50
+ for model in known_models:
51
+ try:
52
+ model_info = litellm.get_model_info(
53
+ model=model, custom_llm_provider=custom_llm_provider
54
+ )
55
+ except Exception:
56
+ continue
57
+ if model_info.get("mode") != "chat":
58
+ continue
59
+ _cost = model_info.get("input_cost_per_token", 0) + model_info.get(
60
+ "output_cost_per_token", 0
61
+ )
62
+ model_costs.append((model, _cost))
63
+
64
+ # Sort by cost (ascending)
65
+ model_costs.sort(key=lambda x: x[1])
66
+
67
+ # Return the top n cheapest models
68
+ return [model for model, _ in model_costs[:n]]
69
+
70
+
71
+ def get_proxy_server_request_headers(litellm_params: Optional[dict]) -> dict:
72
+ """
73
+ Get the `proxy_server_request` headers from the litellm_params.\
74
+
75
+ Use this if you want to access the request headers made to LiteLLM proxy server.
76
+ """
77
+ if litellm_params is None:
78
+ return {}
79
+
80
+ proxy_request_headers = (
81
+ litellm_params.get("proxy_server_request", {}).get("headers", {}) or {}
82
+ )
83
+
84
+ return proxy_request_headers
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py ADDED
@@ -0,0 +1,833 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ import json
3
+ import time
4
+ import traceback
5
+ from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union, cast
6
+
7
+ import litellm
8
+ from litellm._logging import verbose_logger
9
+ from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
10
+ from litellm.litellm_core_utils.prompt_templates.common_utils import (
11
+ _extract_reasoning_content,
12
+ )
13
+ from litellm.types.llms.databricks import DatabricksTool
14
+ from litellm.types.llms.openai import (
15
+ ChatCompletionThinkingBlock,
16
+ ImageURLListItem,
17
+ OpenAIModerationResponse,
18
+ )
19
+ from litellm.types.utils import (
20
+ ChatCompletionDeltaToolCall,
21
+ ChatCompletionMessageToolCall,
22
+ ChatCompletionRedactedThinkingBlock,
23
+ Choices,
24
+ CompletionTokensDetailsWrapper,
25
+ Delta,
26
+ EmbeddingResponse,
27
+ Function,
28
+ HiddenParams,
29
+ ImageResponse,
30
+ )
31
+ from litellm.types.utils import Logprobs as TextCompletionLogprobs
32
+ from litellm.types.utils import (
33
+ Message,
34
+ ModelResponse,
35
+ ModelResponseStream,
36
+ PromptTokensDetailsWrapper,
37
+ RerankResponse,
38
+ StreamingChoices,
39
+ TextChoices,
40
+ TextCompletionResponse,
41
+ TranscriptionResponse,
42
+ TranscriptionUsageDurationObject,
43
+ TranscriptionUsageTokensObject,
44
+ Usage,
45
+ )
46
+
47
+ from .get_headers import get_response_headers
48
+
49
+ _MESSAGE_FIELDS: frozenset = frozenset(Message.model_fields.keys())
50
+ _CHOICES_FIELDS: frozenset = frozenset(Choices.model_fields.keys())
51
+ _MODEL_RESPONSE_FIELDS: frozenset = frozenset(ModelResponse.model_fields.keys()) | {
52
+ "usage"
53
+ }
54
+
55
+
56
+ def _normalize_images_for_message(
57
+ images: Optional[List[dict]],
58
+ ) -> Optional[List[ImageURLListItem]]:
59
+ """
60
+ Ensure each image has an 'index' field, as required by ImageURLListItem.
61
+ Some providers (e.g. OpenRouter) return images without index.
62
+ """
63
+ if not images:
64
+ return cast(Optional[List[ImageURLListItem]], images)
65
+ normalized: List[ImageURLListItem] = []
66
+ for i, img in enumerate(images):
67
+ if isinstance(img, dict) and "index" not in img:
68
+ normalized.append(cast(ImageURLListItem, {**img, "index": i}))
69
+ else:
70
+ normalized.append(cast(ImageURLListItem, img))
71
+ return normalized
72
+
73
+
74
+ def _safe_convert_created_field(created_value) -> int:
75
+ """
76
+ Safely convert a 'created' field value to an integer.
77
+
78
+ Some providers (like SambaNova) return the 'created' field as a float
79
+ (Unix timestamp with fractional seconds), but LiteLLM expects an integer.
80
+
81
+ Args:
82
+ created_value: The value from response_object["created"]
83
+
84
+ Returns:
85
+ int: Unix timestamp as integer
86
+ """
87
+ if created_value is None:
88
+ return int(time.time())
89
+ elif isinstance(created_value, int):
90
+ return created_value
91
+ elif isinstance(created_value, float):
92
+ return int(created_value)
93
+ else:
94
+ # for strings, etc
95
+ try:
96
+ return int(float(created_value))
97
+ except (ValueError, TypeError):
98
+ # Fallback to current time if conversion fails
99
+ return int(time.time())
100
+
101
+
102
+ def convert_tool_call_to_json_mode(
103
+ tool_calls: List[ChatCompletionMessageToolCall],
104
+ convert_tool_call_to_json_mode: bool,
105
+ ) -> Tuple[Optional[Message], Optional[str]]:
106
+ if _should_convert_tool_call_to_json_mode(
107
+ tool_calls=tool_calls,
108
+ convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
109
+ ):
110
+ # to support 'json_schema' logic on older models
111
+ json_mode_content_str: Optional[str] = tool_calls[0]["function"].get(
112
+ "arguments"
113
+ )
114
+ if json_mode_content_str is not None:
115
+ message = litellm.Message(content=json_mode_content_str)
116
+ finish_reason = "stop"
117
+ return message, finish_reason
118
+ return None, None
119
+
120
+
121
+ async def convert_to_streaming_response_async(response_object: Optional[dict] = None):
122
+ """
123
+ Asynchronously converts a response object to a streaming response.
124
+
125
+ Args:
126
+ response_object (Optional[dict]): The response object to be converted. Defaults to None.
127
+
128
+ Raises:
129
+ Exception: If the response object is None.
130
+
131
+ Yields:
132
+ ModelResponse: The converted streaming response object.
133
+
134
+ Returns:
135
+ None
136
+ """
137
+ if response_object is None:
138
+ raise Exception("Error in response object format")
139
+
140
+ model_response_object = ModelResponseStream()
141
+
142
+ if model_response_object is None:
143
+ raise Exception("Error in response creating model response object")
144
+
145
+ choice_list: List[StreamingChoices] = []
146
+
147
+ for idx, choice in enumerate(response_object["choices"]):
148
+ if (
149
+ choice["message"].get("tool_calls", None) is not None
150
+ and isinstance(choice["message"]["tool_calls"], list)
151
+ and len(choice["message"]["tool_calls"]) > 0
152
+ and isinstance(choice["message"]["tool_calls"][0], dict)
153
+ ):
154
+ pydantic_tool_calls = []
155
+ for index, t in enumerate(choice["message"]["tool_calls"]):
156
+ if "index" not in t:
157
+ t["index"] = index
158
+ pydantic_tool_calls.append(ChatCompletionDeltaToolCall(**t))
159
+ choice["message"]["tool_calls"] = pydantic_tool_calls
160
+ delta = Delta(
161
+ content=choice["message"].get("content", None),
162
+ role=choice["message"]["role"],
163
+ function_call=choice["message"].get("function_call", None),
164
+ tool_calls=choice["message"].get("tool_calls", None),
165
+ )
166
+ finish_reason = choice.get("finish_reason", None)
167
+
168
+ if finish_reason is None:
169
+ finish_reason = choice.get("finish_details")
170
+
171
+ logprobs = choice.get("logprobs", None)
172
+
173
+ choice = StreamingChoices(
174
+ finish_reason=finish_reason, index=idx, delta=delta, logprobs=logprobs
175
+ )
176
+ choice_list.append(choice)
177
+
178
+ model_response_object.choices = choice_list
179
+
180
+ if "usage" in response_object and response_object["usage"] is not None:
181
+ setattr(
182
+ model_response_object,
183
+ "usage",
184
+ Usage(
185
+ completion_tokens=response_object["usage"].get("completion_tokens", 0),
186
+ prompt_tokens=response_object["usage"].get("prompt_tokens", 0),
187
+ total_tokens=response_object["usage"].get("total_tokens", 0),
188
+ ),
189
+ )
190
+
191
+ if "id" in response_object:
192
+ model_response_object.id = response_object["id"]
193
+
194
+ if "created" in response_object:
195
+ model_response_object.created = _safe_convert_created_field(
196
+ response_object["created"]
197
+ )
198
+
199
+ if "system_fingerprint" in response_object:
200
+ model_response_object.system_fingerprint = response_object["system_fingerprint"]
201
+
202
+ if "model" in response_object:
203
+ model_response_object.model = response_object["model"]
204
+
205
+ yield model_response_object
206
+ await asyncio.sleep(0)
207
+
208
+
209
+ def convert_to_streaming_response(response_object: Optional[dict] = None):
210
+ # used for yielding Cache hits when stream == True
211
+ if response_object is None:
212
+ raise Exception("Error in response object format")
213
+
214
+ model_response_object = ModelResponseStream()
215
+ choice_list: List[StreamingChoices] = []
216
+ for idx, choice in enumerate(response_object["choices"]):
217
+ delta = Delta(**choice["message"])
218
+ finish_reason = choice.get("finish_reason", None)
219
+ if finish_reason is None:
220
+ # gpt-4 vision can return 'finish_reason' or 'finish_details'
221
+ finish_reason = choice.get("finish_details")
222
+ logprobs = choice.get("logprobs", None)
223
+ enhancements = choice.get("enhancements", None)
224
+ choice = StreamingChoices(
225
+ finish_reason=finish_reason,
226
+ index=idx,
227
+ delta=delta,
228
+ logprobs=logprobs,
229
+ enhancements=enhancements,
230
+ )
231
+
232
+ choice_list.append(choice)
233
+ model_response_object.choices = choice_list
234
+
235
+ if "usage" in response_object and response_object["usage"] is not None:
236
+ setattr(model_response_object, "usage", Usage())
237
+ model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) # type: ignore
238
+ model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) # type: ignore
239
+ model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0) # type: ignore
240
+
241
+ if "id" in response_object:
242
+ model_response_object.id = response_object["id"]
243
+
244
+ if "created" in response_object:
245
+ model_response_object.created = _safe_convert_created_field(
246
+ response_object["created"]
247
+ )
248
+
249
+ if "system_fingerprint" in response_object:
250
+ model_response_object.system_fingerprint = response_object["system_fingerprint"]
251
+
252
+ if "model" in response_object:
253
+ model_response_object.model = response_object["model"]
254
+ yield model_response_object
255
+
256
+
257
+ from collections import defaultdict
258
+
259
+
260
+ def _handle_invalid_parallel_tool_calls(
261
+ tool_calls: List[ChatCompletionMessageToolCall],
262
+ ):
263
+ """
264
+ Handle hallucinated parallel tool call from openai - https://community.openai.com/t/model-tries-to-call-unknown-function-multi-tool-use-parallel/490653
265
+
266
+ Code modified from: https://github.com/phdowling/openai_multi_tool_use_parallel_patch/blob/main/openai_multi_tool_use_parallel_patch.py
267
+ """
268
+
269
+ if tool_calls is None:
270
+ return
271
+ try:
272
+ replacements: Dict[int, List[ChatCompletionMessageToolCall]] = defaultdict(list)
273
+ for i, tool_call in enumerate(tool_calls):
274
+ current_function = tool_call.function.name
275
+ function_args = json.loads(tool_call.function.arguments)
276
+ if current_function == "multi_tool_use.parallel":
277
+ verbose_logger.debug(
278
+ "OpenAI did a weird pseudo-multi-tool-use call, fixing call structure.."
279
+ )
280
+ for _fake_i, _fake_tool_use in enumerate(function_args["tool_uses"]):
281
+ _function_args = _fake_tool_use["parameters"]
282
+ _current_function = _fake_tool_use["recipient_name"]
283
+ if _current_function.startswith("functions."):
284
+ _current_function = _current_function[len("functions.") :]
285
+
286
+ fixed_tc = ChatCompletionMessageToolCall(
287
+ id=f"{tool_call.id}_{_fake_i}",
288
+ type="function",
289
+ function=Function(
290
+ name=_current_function, arguments=json.dumps(_function_args)
291
+ ),
292
+ )
293
+ replacements[i].append(fixed_tc)
294
+
295
+ shift = 0
296
+ for i, replacement in replacements.items():
297
+ tool_calls[:] = (
298
+ tool_calls[: i + shift] + replacement + tool_calls[i + shift + 1 :]
299
+ )
300
+ shift += len(replacement)
301
+
302
+ return tool_calls
303
+ except json.JSONDecodeError:
304
+ # if there is a JSONDecodeError, return the original tool_calls
305
+ return tool_calls
306
+
307
+
308
+ class LiteLLMResponseObjectHandler:
309
+ @staticmethod
310
+ def convert_to_image_response(
311
+ response_object: dict,
312
+ model_response_object: Optional[ImageResponse] = None,
313
+ hidden_params: Optional[dict] = None,
314
+ ) -> ImageResponse:
315
+ response_object.update({"hidden_params": hidden_params})
316
+
317
+ # Handle gpt-image-1 usage field with None values
318
+ if "usage" in response_object and response_object["usage"] is not None:
319
+ usage = response_object["usage"]
320
+ # Check if usage fields are None and provide defaults
321
+ if usage.get("input_tokens") is None:
322
+ usage["input_tokens"] = 0
323
+ if usage.get("output_tokens") is None:
324
+ usage["output_tokens"] = 0
325
+ if usage.get("total_tokens") is None:
326
+ usage["total_tokens"] = usage["input_tokens"] + usage["output_tokens"]
327
+ if usage.get("input_tokens_details") is None:
328
+ usage["input_tokens_details"] = {
329
+ "image_tokens": 0,
330
+ "text_tokens": 0,
331
+ }
332
+
333
+ # Map Responses API naming to Chat Completions API naming for cost calculator
334
+ if usage.get("prompt_tokens") is None:
335
+ usage["prompt_tokens"] = usage.get("input_tokens", 0)
336
+ if usage.get("completion_tokens") is None:
337
+ usage["completion_tokens"] = usage.get("output_tokens", 0)
338
+
339
+ # Convert dicts to wrapper objects so getattr() works in cost calculation
340
+ if isinstance(usage.get("input_tokens_details"), dict):
341
+ usage["prompt_tokens_details"] = PromptTokensDetailsWrapper(
342
+ **usage["input_tokens_details"]
343
+ )
344
+ if isinstance(usage.get("output_tokens_details"), dict):
345
+ usage["completion_tokens_details"] = CompletionTokensDetailsWrapper(
346
+ **usage["output_tokens_details"]
347
+ )
348
+
349
+ if model_response_object is None:
350
+ model_response_object = ImageResponse(**response_object)
351
+ return model_response_object
352
+ else:
353
+ model_response_dict = model_response_object.model_dump()
354
+
355
+ model_response_dict.update(response_object)
356
+ model_response_object = ImageResponse(**model_response_dict)
357
+ return model_response_object
358
+
359
+ @staticmethod
360
+ def convert_to_moderation_response(
361
+ response_object: dict,
362
+ ) -> OpenAIModerationResponse:
363
+ return OpenAIModerationResponse(**response_object)
364
+
365
+ @staticmethod
366
+ def convert_chat_to_text_completion(
367
+ response: ModelResponse,
368
+ text_completion_response: TextCompletionResponse,
369
+ custom_llm_provider: Optional[str] = None,
370
+ ) -> TextCompletionResponse:
371
+ """
372
+ Converts a chat completion response to a text completion response format.
373
+
374
+ Note: This is used for huggingface. For OpenAI / Azure Text the providers files directly return TextCompletionResponse which we then send to user
375
+
376
+ Args:
377
+ response (ModelResponse): The chat completion response to convert
378
+
379
+ Returns:
380
+ TextCompletionResponse: The converted text completion response
381
+
382
+ Example:
383
+ chat_response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi"}])
384
+ text_response = convert_chat_to_text_completion(chat_response)
385
+ """
386
+ transformed_logprobs = LiteLLMResponseObjectHandler._convert_provider_response_logprobs_to_text_completion_logprobs(
387
+ response=response,
388
+ custom_llm_provider=custom_llm_provider,
389
+ )
390
+
391
+ text_completion_response["id"] = response.get("id", None)
392
+ text_completion_response["object"] = "text_completion"
393
+ text_completion_response["created"] = response.get("created", None)
394
+ text_completion_response["model"] = response.get("model", None)
395
+ choices_list: List[TextChoices] = []
396
+
397
+ # Convert each choice to TextChoices
398
+ for choice in response["choices"]:
399
+ text_choices = TextChoices()
400
+ text_choices["text"] = choice["message"]["content"]
401
+ text_choices["index"] = choice["index"]
402
+ text_choices["logprobs"] = transformed_logprobs
403
+ text_choices["finish_reason"] = choice["finish_reason"]
404
+ choices_list.append(text_choices)
405
+
406
+ text_completion_response["choices"] = choices_list
407
+ text_completion_response["usage"] = response.get("usage", None)
408
+ text_completion_response._hidden_params = HiddenParams(
409
+ **response._hidden_params
410
+ )
411
+ return text_completion_response
412
+
413
+ @staticmethod
414
+ def _convert_provider_response_logprobs_to_text_completion_logprobs(
415
+ response: ModelResponse,
416
+ custom_llm_provider: Optional[str] = None,
417
+ ) -> Optional[TextCompletionLogprobs]:
418
+ """
419
+ Convert logprobs from provider to OpenAI.Completion() format
420
+
421
+ Only supported for HF TGI models
422
+ """
423
+ transformed_logprobs: Optional[TextCompletionLogprobs] = None
424
+
425
+ return transformed_logprobs
426
+
427
+
428
+ def _should_convert_tool_call_to_json_mode(
429
+ tool_calls: Optional[
430
+ Union[List[ChatCompletionMessageToolCall], List[DatabricksTool]]
431
+ ] = None,
432
+ convert_tool_call_to_json_mode: Optional[bool] = None,
433
+ ) -> bool:
434
+ """
435
+ Determine if tool calls should be converted to JSON mode
436
+ """
437
+ if (
438
+ convert_tool_call_to_json_mode
439
+ and tool_calls is not None
440
+ and len(tool_calls) == 1
441
+ and tool_calls[0]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME
442
+ ):
443
+ return True
444
+ return False
445
+
446
+
447
+ def convert_to_model_response_object( # noqa: PLR0915
448
+ response_object: Optional[dict] = None,
449
+ model_response_object: Optional[
450
+ Union[
451
+ ModelResponse,
452
+ EmbeddingResponse,
453
+ ImageResponse,
454
+ TranscriptionResponse,
455
+ RerankResponse,
456
+ ]
457
+ ] = None,
458
+ response_type: Literal[
459
+ "completion", "embedding", "image_generation", "audio_transcription", "rerank"
460
+ ] = "completion",
461
+ stream=False,
462
+ start_time=None,
463
+ end_time=None,
464
+ hidden_params: Optional[dict] = None,
465
+ _response_headers: Optional[dict] = None,
466
+ convert_tool_call_to_json_mode: Optional[
467
+ bool
468
+ ] = None, # used for supporting 'json_schema' on older models
469
+ ):
470
+ additional_headers = get_response_headers(_response_headers)
471
+
472
+ if hidden_params is None:
473
+ hidden_params = {}
474
+
475
+ # Preserve existing additional_headers if they contain important provider headers
476
+ # For responses API, additional_headers may already be set with LLM provider headers
477
+ existing_additional_headers = hidden_params.get("additional_headers", {})
478
+ if existing_additional_headers and _response_headers is None:
479
+ # Keep existing headers when _response_headers is None (responses API case)
480
+ additional_headers = existing_additional_headers
481
+ else:
482
+ # Merge new headers with existing ones
483
+ if existing_additional_headers:
484
+ additional_headers.update(existing_additional_headers)
485
+
486
+ hidden_params["additional_headers"] = additional_headers
487
+
488
+ ### CHECK IF ERROR IN RESPONSE ### - openrouter returns these in the dictionary
489
+ # Some OpenAI-compatible providers (e.g., Apertis) return empty error objects
490
+ # even on success. Only raise if the error contains meaningful data.
491
+ if (
492
+ response_object is not None
493
+ and "error" in response_object
494
+ and response_object["error"] is not None
495
+ ):
496
+ error_obj = response_object["error"]
497
+ has_meaningful_error = False
498
+
499
+ if isinstance(error_obj, dict):
500
+ # Check if error dict has non-empty message or non-null code
501
+ error_message = error_obj.get("message", "")
502
+ error_code = error_obj.get("code")
503
+ has_meaningful_error = bool(error_message) or error_code is not None
504
+ elif isinstance(error_obj, str):
505
+ # String error is meaningful if non-empty
506
+ has_meaningful_error = bool(error_obj)
507
+ else:
508
+ # Any other truthy value is considered meaningful
509
+ has_meaningful_error = True
510
+
511
+ if has_meaningful_error:
512
+ error_args = {"status_code": 422, "message": "Error in response object"}
513
+ if isinstance(error_obj, dict):
514
+ if "code" in error_obj:
515
+ error_args["status_code"] = error_obj["code"]
516
+ if "message" in error_obj:
517
+ if isinstance(error_obj["message"], dict):
518
+ message_str = json.dumps(error_obj["message"])
519
+ else:
520
+ message_str = str(error_obj["message"])
521
+ error_args["message"] = message_str
522
+ raised_exception = Exception()
523
+ setattr(raised_exception, "status_code", error_args["status_code"])
524
+ setattr(raised_exception, "message", error_args["message"])
525
+ raise raised_exception
526
+
527
+ try:
528
+ if response_type == "completion" and (
529
+ model_response_object is None
530
+ or isinstance(model_response_object, ModelResponse)
531
+ ):
532
+ if response_object is None or model_response_object is None:
533
+ raise Exception("Error in response object format")
534
+ if stream is True:
535
+ # for returning cached responses, we need to yield a generator
536
+ return convert_to_streaming_response(response_object=response_object)
537
+ choice_list: List[Choices] = []
538
+
539
+ assert response_object["choices"] is not None and isinstance(
540
+ response_object["choices"], Iterable
541
+ )
542
+
543
+ for idx, choice in enumerate(response_object["choices"]):
544
+ ## HANDLE JSON MODE - anthropic returns single function call]
545
+ tool_calls = choice["message"].get("tool_calls", None)
546
+ if tool_calls is not None:
547
+ _openai_tool_calls = []
548
+ for _tc in tool_calls:
549
+ _openai_tc = ChatCompletionMessageToolCall(**_tc)
550
+ _openai_tool_calls.append(_openai_tc)
551
+ fixed_tool_calls = _handle_invalid_parallel_tool_calls(
552
+ _openai_tool_calls
553
+ )
554
+
555
+ if fixed_tool_calls is not None:
556
+ tool_calls = fixed_tool_calls
557
+
558
+ message: Optional[Message] = None
559
+ finish_reason: Optional[str] = None
560
+ if _should_convert_tool_call_to_json_mode(
561
+ tool_calls=tool_calls,
562
+ convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
563
+ ):
564
+ # to support 'json_schema' logic on older models
565
+ json_mode_content_str: Optional[str] = tool_calls[0][
566
+ "function"
567
+ ].get("arguments")
568
+ if json_mode_content_str is not None:
569
+ message = litellm.Message(content=json_mode_content_str)
570
+ finish_reason = "stop"
571
+ if message is None:
572
+ # Preserve provider_specific_fields if already present
573
+ # in the response (e.g. from proxy passthrough)
574
+ provider_specific_fields = dict(
575
+ choice["message"].get("provider_specific_fields", None) or {}
576
+ )
577
+ for f in choice["message"].keys() - _MESSAGE_FIELDS:
578
+ provider_specific_fields[f] = choice["message"][f]
579
+
580
+ # Handle reasoning models that display `reasoning_content` within `content`
581
+ reasoning_content, content = _extract_reasoning_content(
582
+ choice["message"]
583
+ )
584
+
585
+ # Handle thinking models that display `thinking_blocks` within `content`
586
+ thinking_blocks: Optional[
587
+ List[
588
+ Union[
589
+ ChatCompletionThinkingBlock,
590
+ ChatCompletionRedactedThinkingBlock,
591
+ ]
592
+ ]
593
+ ] = None
594
+ if "thinking_blocks" in choice["message"]:
595
+ thinking_blocks = choice["message"]["thinking_blocks"]
596
+ provider_specific_fields["thinking_blocks"] = thinking_blocks
597
+
598
+ if reasoning_content:
599
+ provider_specific_fields[
600
+ "reasoning_content"
601
+ ] = reasoning_content
602
+
603
+ message = Message(
604
+ content=content,
605
+ role=choice["message"]["role"] or "assistant",
606
+ function_call=choice["message"].get("function_call", None),
607
+ tool_calls=tool_calls,
608
+ audio=choice["message"].get("audio", None),
609
+ provider_specific_fields=provider_specific_fields,
610
+ reasoning_content=reasoning_content,
611
+ thinking_blocks=thinking_blocks,
612
+ annotations=choice["message"].get("annotations", None),
613
+ images=_normalize_images_for_message(
614
+ choice["message"].get("images", None)
615
+ ),
616
+ )
617
+ finish_reason = choice.get("finish_reason", None)
618
+ if finish_reason is None:
619
+ # gpt-4 vision can return 'finish_reason' or 'finish_details'
620
+ finish_reason = choice.get("finish_details") or "stop"
621
+ if (
622
+ finish_reason == "stop"
623
+ and message.tool_calls
624
+ and len(message.tool_calls) > 0
625
+ ):
626
+ finish_reason = "tool_calls"
627
+
628
+ ## PROVIDER SPECIFIC FIELDS ##
629
+ provider_specific_fields = {
630
+ f: choice[f] for f in choice.keys() - _CHOICES_FIELDS
631
+ }
632
+
633
+ logprobs = choice.get("logprobs", None)
634
+ enhancements = choice.get("enhancements", None)
635
+ choice = Choices(
636
+ finish_reason=finish_reason,
637
+ index=idx,
638
+ message=message,
639
+ logprobs=logprobs,
640
+ enhancements=enhancements,
641
+ provider_specific_fields=provider_specific_fields,
642
+ )
643
+ choice_list.append(choice)
644
+ model_response_object.choices = choice_list # type: ignore
645
+
646
+ if "usage" in response_object and response_object["usage"] is not None:
647
+ usage_object = litellm.Usage(**response_object["usage"])
648
+ setattr(model_response_object, "usage", usage_object)
649
+ if "created" in response_object:
650
+ model_response_object.created = _safe_convert_created_field(
651
+ response_object["created"]
652
+ )
653
+
654
+ if "id" in response_object:
655
+ # Preserve the auto-generated id from ModelResponse.__init__
656
+ # when the provider returns a falsy id (None, "")
657
+ model_response_object.id = (
658
+ response_object["id"] or model_response_object.id
659
+ )
660
+
661
+ if "system_fingerprint" in response_object:
662
+ model_response_object.system_fingerprint = response_object[
663
+ "system_fingerprint"
664
+ ]
665
+
666
+ if "model" in response_object:
667
+ if model_response_object.model is None:
668
+ model_response_object.model = response_object["model"]
669
+ elif (
670
+ "/" in model_response_object.model
671
+ and response_object["model"] is not None
672
+ ):
673
+ openai_compatible_provider = model_response_object.model.split("/")[
674
+ 0
675
+ ]
676
+ model_response_object.model = (
677
+ openai_compatible_provider + "/" + response_object["model"]
678
+ )
679
+
680
+ if start_time is not None and end_time is not None:
681
+ if isinstance(start_time, type(end_time)):
682
+ model_response_object._response_ms = ( # type: ignore
683
+ end_time - start_time
684
+ ).total_seconds() * 1000
685
+
686
+ if hidden_params is not None:
687
+ if model_response_object._hidden_params is None:
688
+ model_response_object._hidden_params = {}
689
+ model_response_object._hidden_params.update(hidden_params)
690
+
691
+ if _response_headers is not None:
692
+ model_response_object._response_headers = _response_headers
693
+
694
+ for k, v in response_object.items():
695
+ if k not in _MODEL_RESPONSE_FIELDS:
696
+ setattr(model_response_object, k, v)
697
+
698
+ return model_response_object
699
+ elif response_type == "embedding" and (
700
+ model_response_object is None
701
+ or isinstance(model_response_object, EmbeddingResponse)
702
+ ):
703
+ if response_object is None:
704
+ raise Exception("Error in response object format")
705
+
706
+ if model_response_object is None:
707
+ model_response_object = EmbeddingResponse()
708
+
709
+ if "model" in response_object:
710
+ model_response_object.model = response_object["model"]
711
+
712
+ if "object" in response_object:
713
+ model_response_object.object = response_object["object"]
714
+
715
+ model_response_object.data = response_object["data"]
716
+
717
+ if "usage" in response_object and response_object["usage"] is not None:
718
+ model_response_object.usage.completion_tokens = response_object["usage"].get("completion_tokens", 0) # type: ignore
719
+ model_response_object.usage.prompt_tokens = response_object["usage"].get("prompt_tokens", 0) # type: ignore
720
+ model_response_object.usage.total_tokens = response_object["usage"].get("total_tokens", 0) # type: ignore
721
+
722
+ if start_time is not None and end_time is not None:
723
+ model_response_object._response_ms = ( # type: ignore
724
+ end_time - start_time
725
+ ).total_seconds() * 1000 # return response latency in ms like openai
726
+
727
+ if hidden_params is not None:
728
+ model_response_object._hidden_params = hidden_params
729
+
730
+ if _response_headers is not None:
731
+ model_response_object._response_headers = _response_headers
732
+
733
+ return model_response_object
734
+ elif response_type == "image_generation" and (
735
+ model_response_object is None
736
+ or isinstance(model_response_object, ImageResponse)
737
+ ):
738
+ if response_object is None:
739
+ raise Exception("Error in response object format")
740
+
741
+ return LiteLLMResponseObjectHandler.convert_to_image_response(
742
+ response_object=response_object,
743
+ model_response_object=model_response_object,
744
+ hidden_params=hidden_params,
745
+ )
746
+
747
+ elif response_type == "audio_transcription" and (
748
+ model_response_object is None
749
+ or isinstance(model_response_object, TranscriptionResponse)
750
+ ):
751
+ if response_object is None:
752
+ raise Exception("Error in response object format")
753
+
754
+ if model_response_object is None:
755
+ model_response_object = TranscriptionResponse()
756
+
757
+ if "text" in response_object:
758
+ model_response_object.text = response_object["text"]
759
+
760
+ optional_keys = ["language", "task", "duration", "words", "segments"]
761
+ for key in optional_keys: # not guaranteed to be in response
762
+ if key in response_object:
763
+ setattr(model_response_object, key, response_object[key])
764
+
765
+ if "usage" in response_object and response_object["usage"] is not None:
766
+ tr_usage_object: Optional[
767
+ Union[
768
+ TranscriptionUsageDurationObject, TranscriptionUsageTokensObject
769
+ ]
770
+ ] = None
771
+
772
+ if response_object["usage"].get("type", None) == "duration":
773
+ tr_usage_object = TranscriptionUsageDurationObject(
774
+ **response_object["usage"]
775
+ )
776
+ elif response_object["usage"].get("type", None) == "tokens":
777
+ tr_usage_object = TranscriptionUsageTokensObject(
778
+ **response_object["usage"]
779
+ )
780
+ if tr_usage_object is not None:
781
+ setattr(model_response_object, "usage", tr_usage_object)
782
+
783
+ if hidden_params is not None:
784
+ model_response_object._hidden_params = hidden_params
785
+
786
+ # Store internally-calculated duration in _hidden_params for cost
787
+ # tracking without exposing it in the response body. Must be set
788
+ # after hidden_params assignment to avoid being overwritten.
789
+ if "_audio_transcription_duration" in response_object:
790
+ model_response_object._hidden_params[
791
+ "audio_transcription_duration"
792
+ ] = response_object["_audio_transcription_duration"]
793
+
794
+ if _response_headers is not None:
795
+ model_response_object._response_headers = _response_headers
796
+
797
+ return model_response_object
798
+ elif response_type == "rerank" and (
799
+ model_response_object is None
800
+ or isinstance(model_response_object, RerankResponse)
801
+ ):
802
+ if response_object is None:
803
+ raise Exception("Error in response object format")
804
+
805
+ if model_response_object is None:
806
+ model_response_object = RerankResponse(**response_object)
807
+ return model_response_object
808
+
809
+ if "id" in response_object:
810
+ model_response_object.id = response_object["id"]
811
+
812
+ if "meta" in response_object:
813
+ model_response_object.meta = response_object["meta"]
814
+
815
+ if "results" in response_object:
816
+ model_response_object.results = response_object["results"]
817
+
818
+ return model_response_object
819
+ except Exception:
820
+ received_args = dict(
821
+ response_object=response_object,
822
+ model_response_object=model_response_object,
823
+ response_type=response_type,
824
+ stream=stream,
825
+ start_time=start_time,
826
+ end_time=end_time,
827
+ hidden_params=hidden_params,
828
+ _response_headers=_response_headers,
829
+ convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
830
+ )
831
+ raise Exception(
832
+ f"Invalid response object {traceback.format_exc()}\n\nreceived_args={received_args}"
833
+ )
loan-advisor-agent/.venv/Lib/site-packages/litellm/litellm_core_utils/llm_response_utils/get_api_base.py ADDED
@@ -0,0 +1,119 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import Optional, Union
2
+
3
+ import litellm
4
+ from litellm import verbose_logger
5
+
6
+ from ...litellm_core_utils.get_llm_provider_logic import get_llm_provider
7
+ from ...types.router import LiteLLM_Params
8
+
9
+
10
+ def get_api_base(
11
+ model: str, optional_params: Union[dict, LiteLLM_Params]
12
+ ) -> Optional[str]:
13
+ """
14
+ Returns the api base used for calling the model.
15
+
16
+ Parameters:
17
+ - model: str - the model passed to litellm.completion()
18
+ - optional_params - the 'litellm_params' in router.completion *OR* additional params passed to litellm.completion - eg. api_base, api_key, etc. See `LiteLLM_Params` - https://github.com/BerriAI/litellm/blob/f09e6ba98d65e035a79f73bc069145002ceafd36/litellm/router.py#L67
19
+
20
+ Returns:
21
+ - string (api_base) or None
22
+
23
+ Example:
24
+ ```
25
+ from litellm import get_api_base
26
+
27
+ get_api_base(model="gemini/gemini-pro")
28
+ ```
29
+ """
30
+
31
+ try:
32
+ if isinstance(optional_params, LiteLLM_Params):
33
+ _optional_params = optional_params
34
+ elif "model" in optional_params:
35
+ _optional_params = LiteLLM_Params(**optional_params)
36
+ else: # prevent needing to copy and pop the dict
37
+ _optional_params = LiteLLM_Params(
38
+ model=model, **optional_params
39
+ ) # convert to pydantic object
40
+ except Exception:
41
+ return None
42
+ # get llm provider
43
+
44
+ if _optional_params.api_base is not None:
45
+ return _optional_params.api_base
46
+
47
+ if litellm.model_alias_map and model in litellm.model_alias_map:
48
+ model = litellm.model_alias_map[model]
49
+ try:
50
+ (
51
+ model,
52
+ custom_llm_provider,
53
+ dynamic_api_key,
54
+ dynamic_api_base,
55
+ ) = get_llm_provider(
56
+ model=model,
57
+ custom_llm_provider=_optional_params.custom_llm_provider,
58
+ api_base=_optional_params.api_base,
59
+ api_key=_optional_params.api_key,
60
+ )
61
+ except Exception as e:
62
+ verbose_logger.debug("Error occurred in getting api base - {}".format(str(e)))
63
+ custom_llm_provider = None
64
+ dynamic_api_base = None
65
+
66
+ if dynamic_api_base is not None:
67
+ return dynamic_api_base
68
+
69
+ stream: bool = getattr(optional_params, "stream", False)
70
+
71
+ if (
72
+ _optional_params.vertex_location is not None
73
+ and _optional_params.vertex_project is not None
74
+ ):
75
+ from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
76
+ from litellm.types.llms.vertex_ai import VertexPartnerProvider
77
+
78
+ if "claude" in model:
79
+ _api_base = VertexBase.create_vertex_url(
80
+ vertex_location=_optional_params.vertex_location,
81
+ vertex_project=_optional_params.vertex_project,
82
+ model=model,
83
+ stream=stream,
84
+ partner=VertexPartnerProvider.claude,
85
+ )
86
+ else:
87
+ if stream:
88
+ _api_base = "{}-aiplatform.googleapis.com/v1/projects/{}/locations/{}/publishers/google/models/{}:streamGenerateContent".format(
89
+ _optional_params.vertex_location,
90
+ _optional_params.vertex_project,
91
+ _optional_params.vertex_location,
92
+ model,
93
+ )
94
+ else:
95
+ _api_base = "{}-aiplatform.googleapis.com/v1/projects/{}/locations/{}/publishers/google/models/{}:generateContent".format(
96
+ _optional_params.vertex_location,
97
+ _optional_params.vertex_project,
98
+ _optional_params.vertex_location,
99
+ model,
100
+ )
101
+ return _api_base
102
+
103
+ if custom_llm_provider is None:
104
+ return None
105
+
106
+ if custom_llm_provider == "gemini":
107
+ if stream:
108
+ _api_base = "https://generativelanguage.googleapis.com/v1beta/models/{}:streamGenerateContent".format(
109
+ model
110
+ )
111
+ else:
112
+ _api_base = "https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent".format(
113
+ model
114
+ )
115
+ return _api_base
116
+ elif custom_llm_provider == "openai":
117
+ _api_base = "https://api.openai.com"
118
+ return _api_base
119
+ return None