diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/README.md b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..7a81367929b0feaa462bf673d577b74af15b7e16
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/README.md
@@ -0,0 +1,12 @@
+## File Structure
+
+### August 27th, 2024
+
+To make it easy to see how calls are transformed for each model/provider:
+
+we are working on moving all supported litellm providers to a folder structure, where folder name is the supported litellm provider name.
+
+Each folder will contain a `*_transformation.py` file, which has all the request/response transformation logic, making it easy to see how calls are modified.
+
+E.g. `cohere/`, `bedrock/`.
+
\ No newline at end of file
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..cf6a6ed7a5432af1f99b24aefbd185e42bbff999
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/chat/transformation.py
@@ -0,0 +1,148 @@
+"""
+Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
+"""
+
+from typing import TYPE_CHECKING, List, Optional, Tuple
+
+from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
+from litellm.secret_managers.main import get_secret_bool, get_secret_str
+from litellm.types.router import LiteLLM_Params
+
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+
+if TYPE_CHECKING:
+ from litellm.types.llms.openai import AllMessageValues
+
+
+class LiteLLMProxyChatConfig(OpenAIGPTConfig):
+ def get_supported_openai_params(self, model: str) -> List:
+ params_list = super().get_supported_openai_params(model)
+ params_list.extend(OPENAI_CHAT_COMPLETION_PARAMS)
+ return params_list
+
+ def _map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ supported_openai_params = self.get_supported_openai_params(model)
+ for param, value in non_default_params.items():
+ if param == "thinking":
+ optional_params.setdefault("extra_body", {})["thinking"] = value
+ elif param in supported_openai_params:
+ optional_params[param] = value
+ return optional_params
+
+ def _get_openai_compatible_provider_info(
+ self, api_base: Optional[str], api_key: Optional[str]
+ ) -> Tuple[Optional[str], Optional[str]]:
+ api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE") # type: ignore
+ dynamic_api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY")
+ return api_base, dynamic_api_key
+
+ def get_models(
+ self, api_key: Optional[str] = None, api_base: Optional[str] = None
+ ) -> List[str]:
+ api_base, api_key = self._get_openai_compatible_provider_info(api_base, api_key)
+ if api_base is None:
+ raise ValueError(
+ "api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`"
+ )
+ models = super().get_models(api_key=api_key, api_base=api_base)
+ return [f"litellm_proxy/{model}" for model in models]
+
+ @staticmethod
+ def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
+ return api_key or get_secret_str("LITELLM_PROXY_API_KEY")
+
+ @staticmethod
+ def _should_use_litellm_proxy_by_default(
+ litellm_params: Optional[LiteLLM_Params] = None,
+ ):
+ """
+ Returns True if litellm proxy should be used by default for a given request
+
+ Issue: https://github.com/BerriAI/litellm/issues/10559
+
+ Use case:
+ - When using Google ADK, users want a flag to dynamically enable sending the request to litellm proxy or not
+ - Allow the model name to be passed in original format and still use litellm proxy:
+ "gemini/gemini-1.5-pro", "openai/gpt-4", "mistral/llama-2-70b-chat" etc.
+ """
+ import litellm
+
+ if get_secret_bool("USE_LITELLM_PROXY") is True:
+ return True
+ if litellm_params and litellm_params.use_litellm_proxy is True:
+ return True
+ if litellm.use_litellm_proxy is True:
+ return True
+ return False
+
+ @staticmethod
+ def litellm_proxy_get_custom_llm_provider_info(
+ model: str, api_base: Optional[str] = None, api_key: Optional[str] = None
+ ) -> Tuple[str, str, Optional[str], Optional[str]]:
+ """
+ Force use litellm proxy for all models
+
+ Issue: https://github.com/BerriAI/litellm/issues/10559
+
+ Expected behavior:
+ - custom_llm_provider will be 'litellm_proxy'
+ - api_base = api_base OR LITELLM_PROXY_API_BASE
+ - api_key = api_key OR LITELLM_PROXY_API_KEY
+
+ Use case:
+ - When using Google ADK, users want a flag to dynamically enable sending the request to litellm proxy or not
+ - Allow the model name to be passed in original format and still use litellm proxy:
+ "gemini/gemini-1.5-pro", "openai/gpt-4", "mistral/llama-2-70b-chat" etc.
+
+ Return model, custom_llm_provider, dynamic_api_key, api_base
+ """
+ import litellm
+
+ custom_llm_provider = "litellm_proxy"
+ if model.startswith("litellm_proxy/"):
+ model = model.split("/", 1)[1]
+
+ (
+ api_base,
+ api_key,
+ ) = litellm.LiteLLMProxyChatConfig()._get_openai_compatible_provider_info(
+ api_base=api_base, api_key=api_key
+ )
+
+ return model, custom_llm_provider, api_key, api_base
+
+ def transform_request(
+ self,
+ model: str,
+ messages: List["AllMessageValues"],
+ optional_params: dict,
+ litellm_params: dict,
+ headers: dict,
+ ) -> dict:
+ # don't transform the request
+ return {
+ "model": model,
+ "messages": messages,
+ **optional_params,
+ }
+
+ async def async_transform_request(
+ self,
+ model: str,
+ messages: List["AllMessageValues"],
+ optional_params: dict,
+ litellm_params: dict,
+ headers: dict,
+ ) -> dict:
+ # don't transform the request
+ return {
+ "model": model,
+ "messages": messages,
+ **optional_params,
+ }
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/image_edit/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/image_edit/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..5f5e2bdb24df0047e7193b52536de4937181f226
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/image_edit/transformation.py
@@ -0,0 +1,26 @@
+from typing import Optional
+
+from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
+from litellm.secret_managers.main import get_secret_str
+
+
+class LiteLLMProxyImageEditConfig(OpenAIImageEditConfig):
+ """Configuration for image edit requests routed through LiteLLM Proxy."""
+
+ def validate_environment(
+ self, headers: dict, model: str, api_key: Optional[str] = None
+ ) -> dict:
+ api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY")
+ headers.update({"Authorization": f"Bearer {api_key}"})
+ return headers
+
+ def get_complete_url(
+ self, model: str, api_base: Optional[str], litellm_params: dict
+ ) -> str:
+ api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE")
+ if api_base is None:
+ raise ValueError(
+ "api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`"
+ )
+ api_base = api_base.rstrip("/")
+ return f"{api_base}/images/edits"
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/image_generation/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/image_generation/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..3932070e96413b7afd41e04d4c2e5cb2190789b3
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/image_generation/transformation.py
@@ -0,0 +1,41 @@
+from typing import Optional
+
+from litellm.llms.openai.image_generation.gpt_transformation import (
+ GPTImageGenerationConfig,
+)
+from litellm.secret_managers.main import get_secret_str
+
+
+class LiteLLMProxyImageGenerationConfig(GPTImageGenerationConfig):
+ """Configuration for image generation requests routed through LiteLLM Proxy."""
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages,
+ optional_params: dict,
+ litellm_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY")
+ headers.update({"Authorization": f"Bearer {api_key}"})
+ return headers
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ api_key: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: dict,
+ stream: Optional[bool] = None,
+ ) -> str:
+ api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE")
+ if api_base is None:
+ raise ValueError(
+ "api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`"
+ )
+ api_base = api_base.rstrip("/")
+ return f"{api_base}/images/generations"
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/responses/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/responses/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5bbaa78d1d8fff32b5e8576beb01302cee8a799
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/responses/transformation.py
@@ -0,0 +1,52 @@
+"""
+Responses API transformation for LiteLLM Proxy provider.
+
+LiteLLM Proxy supports the OpenAI Responses API natively when the underlying model supports it.
+This config enables pass-through behavior to the proxy's /v1/responses endpoint.
+"""
+
+from typing import Optional
+
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.utils import LlmProviders
+
+
+class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
+ """
+ Configuration for LiteLLM Proxy Responses API support.
+
+ Extends OpenAI's config since the proxy follows OpenAI's API spec,
+ but uses LITELLM_PROXY_API_BASE for the base URL.
+ """
+
+ @property
+ def custom_llm_provider(self) -> LlmProviders:
+ return LlmProviders.LITELLM_PROXY
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ litellm_params: dict,
+ ) -> str:
+ """
+ Get the endpoint for LiteLLM Proxy responses API.
+
+ Uses LITELLM_PROXY_API_BASE environment variable if api_base is not provided.
+ """
+ api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE")
+
+ if api_base is None:
+ raise ValueError(
+ "api_base not set for LiteLLM Proxy responses API. "
+ "Set via api_base parameter or LITELLM_PROXY_API_BASE environment variable"
+ )
+
+ # Remove trailing slashes
+ api_base = api_base.rstrip("/")
+
+ return f"{api_base}/responses"
+
+ def supports_native_websocket(self) -> bool:
+ """LiteLLM Proxy does not support native WebSocket for Responses API"""
+ return False
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/README.md b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..1dfeff1a42cf4842280119fae47fa051e03c8caf
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/README.md
@@ -0,0 +1,381 @@
+# LiteLLM Skills - Database-Backed Skills Storage
+
+This module provides database-backed skills storage as an alternative to Anthropic's cloud-based Skills API. It enables using skills with **any LLM provider** (Bedrock, OpenAI, Azure, etc.) by storing skills locally and converting them to tools + system prompt injection.
+
+## Architecture
+
+```mermaid
+flowchart TB
+ subgraph "Skill Creation"
+ A[User creates skill with ZIP file] --> B{custom_llm_provider?}
+ B -->|anthropic| C[Forward to Anthropic API]
+ B -->|litellm_proxy| D[Store in LiteLLM Database]
+
+ D --> E[Extract & store:
- display_title
- description
- instructions
- file_content ZIP]
+ end
+
+ subgraph "Skill Usage in Messages API"
+ F[Request with container.skills] --> G[SkillsInjectionHook]
+ G --> H{skill_id prefix?}
+
+ H -->|"litellm:skill_abc"| I[Fetch from LiteLLM DB]
+ H -->|"skill_xyz" no prefix| J[Pass to Anthropic as native skill]
+
+ I --> K{Model provider?}
+ K -->|Anthropic API| L[Convert to tools]
+ K -->|Bedrock/OpenAI/etc| M[Convert to tools +
Inject SKILL.md into system prompt]
+
+ J --> N[Keep in container.skills]
+ end
+
+ subgraph "Skill Resolution for Non-Anthropic"
+ M --> O[Extract SKILL.md from ZIP]
+ O --> P[Add to system prompt:
# Available Skills
## Skill: My Skill
SKILL.md content...]
+ P --> Q[Create OpenAI-style tool:
type: function
name: skill_id
description: instructions]
+ Q --> R[Send to LLM Provider]
+ end
+```
+
+## Automatic Code Execution
+
+For skills that include executable code (Python files), LiteLLM automatically handles:
+
+1. **Pre-call hook** (`async_pre_call_hook`): Adds `litellm_code_execution` tool, injects SKILL.md content
+2. **Post-call hook** (`async_post_call_success_deployment_hook`): Detects tool calls, executes code in Docker sandbox, continues loop
+3. **Returns files**: Generated files (GIFs, images, etc.) returned directly on response
+
+```mermaid
+sequenceDiagram
+ participant User
+ participant LiteLLM as LiteLLM SDK
+ participant PreHook as async_pre_call_hook
+ participant LLM as LLM Provider
+ participant PostHook as async_post_call_success_deployment_hook
+ participant Sandbox as Docker Sandbox
+
+ User->>LiteLLM: litellm.acompletion(model, messages, container={skills: [...]})
+
+ Note over LiteLLM,PreHook: PRE-CALL HOOK
+ LiteLLM->>PreHook: Intercept request
+ PreHook->>PreHook: Fetch skill from DB (litellm:skill_id)
+ PreHook->>PreHook: Extract SKILL.md from ZIP
+ PreHook->>PreHook: Inject SKILL.md into system prompt
+ PreHook->>PreHook: Add litellm_code_execution tool
+ PreHook->>PreHook: Store skill files in metadata
+ PreHook-->>LiteLLM: Modified request
+
+ LiteLLM->>LLM: Forward to provider (OpenAI/Bedrock/etc)
+ LLM-->>LiteLLM: Response with tool_calls
+
+ Note over LiteLLM,PostHook: POST-CALL HOOK (Agentic Loop)
+ LiteLLM->>PostHook: Check response
+
+ loop Until no more tool calls
+ PostHook->>PostHook: Check for litellm_code_execution tool call
+ alt Has code execution tool call
+ PostHook->>Sandbox: Execute Python code
+ Sandbox->>Sandbox: Copy skill files to /sandbox
+ Sandbox->>Sandbox: Install requirements.txt
+ Sandbox->>Sandbox: Run code
+ Sandbox-->>PostHook: Result + generated files
+ PostHook->>PostHook: Add tool result to messages
+ PostHook->>LLM: Make another LLM call
+ LLM-->>PostHook: New response
+ else No code execution
+ PostHook->>PostHook: Break loop
+ end
+ end
+
+ PostHook->>PostHook: Attach files to response._litellm_generated_files
+ PostHook-->>LiteLLM: Modified response with files
+ LiteLLM-->>User: Final response with generated files
+```
+
+```python
+import litellm
+from litellm.proxy.hooks.litellm_skills import SkillsInjectionHook
+
+# Register the hook (done once at startup)
+hook = SkillsInjectionHook()
+litellm.callbacks.append(hook)
+
+# ONE request - LiteLLM handles everything automatically
+# The container parameter triggers the SkillsInjectionHook
+response = await litellm.acompletion(
+ model="gpt-4o-mini",
+ messages=[{"role": "user", "content": "Create a bouncing ball GIF"}],
+ container={
+ "skills": [{"type": "custom", "skill_id": "litellm:skill_abc123"}]
+ },
+)
+
+# Files are attached directly to response
+generated_files = response._litellm_generated_files
+for f in generated_files:
+ print(f"Generated: {f['name']} ({f['size']} bytes)")
+ # f['content_base64'] contains the file data
+```
+
+This mimics Anthropic's behavior - no manual agentic loop needed!
+
+### How it works
+
+The `SkillsInjectionHook` uses two hooks:
+
+1. **`async_pre_call_hook`** (proxy only): Transforms the request before LLM call
+ - Fetches skills from DB
+ - Injects SKILL.md into system prompt
+ - Adds `litellm_code_execution` tool
+ - Sets `_litellm_code_execution_enabled=True` in metadata
+
+2. **`async_post_call_success_deployment_hook`** (SDK + proxy): Called after LLM response
+ - Checks if response has `litellm_code_execution` tool call
+ - Executes code in Docker sandbox
+ - Adds result to messages, makes another LLM call
+ - Repeats until model gives final response
+ - Attaches generated files to `response._litellm_generated_files`
+
+## File Structure
+
+```
+litellm/llms/litellm_proxy/skills/
+├── __init__.py # Exports all skill components
+├── handler.py # LiteLLMSkillsHandler - database CRUD operations (Prisma)
+├── transformation.py # LiteLLMSkillsTransformationHandler - SDK transformation layer
+├── prompt_injection.py # SkillPromptInjectionHandler - SKILL.md extraction and injection
+├── sandbox_executor.py # SkillsSandboxExecutor - Docker sandbox code execution
+├── code_execution.py # CodeExecutionHandler - automatic agentic loop
+└── README.md # This file
+
+litellm/proxy/hooks/litellm_skills/
+├── __init__.py # Re-exports from SDK + SkillsInjectionHook
+└── main.py # SkillsInjectionHook - CustomLogger hook for proxy
+```
+
+## Components
+
+### 1. `handler.py` - LiteLLMSkillsHandler
+
+Database operations for skills CRUD:
+
+```python
+from litellm.llms.litellm_proxy.skills import LiteLLMSkillsHandler
+
+# Create skill
+skill = await LiteLLMSkillsHandler.create_skill(
+ data=NewSkillRequest(
+ display_title="My Skill",
+ description="A helpful skill",
+ instructions="Use this skill when...",
+ file_content=zip_bytes, # ZIP file content
+ file_name="my-skill.zip",
+ file_type="application/zip",
+ ),
+ user_id="user_123"
+)
+
+# List skills
+skills = await LiteLLMSkillsHandler.list_skills(limit=10, offset=0)
+
+# Get skill
+skill = await LiteLLMSkillsHandler.get_skill(skill_id="skill_abc123")
+
+# Delete skill
+await LiteLLMSkillsHandler.delete_skill(skill_id="skill_abc123")
+```
+
+### 2. `transformation.py` - LiteLLMSkillsTransformationHandler
+
+SDK-level transformation layer that wraps handler operations:
+
+```python
+from litellm.llms.litellm_proxy.skills import LiteLLMSkillsTransformationHandler
+
+handler = LiteLLMSkillsTransformationHandler()
+
+# Async create
+skill = await handler.create_skill_handler(
+ display_title="My Skill",
+ files=[zip_file],
+ _is_async=True
+)
+```
+
+## Skill ZIP Format
+
+Skills must be packaged as ZIP files with a `SKILL.md` file:
+
+```
+my-skill.zip
+└── my-skill/
+ └── SKILL.md
+```
+
+### SKILL.md Format
+
+```markdown
+---
+name: my-skill
+description: A brief description of what this skill does
+---
+
+# My Skill
+
+Detailed instructions for the LLM on how to use this skill.
+
+## Usage
+
+When the user asks about X, use this skill to...
+
+## Examples
+
+- Example 1: ...
+- Example 2: ...
+```
+
+## SDK Usage
+
+### Create Skill in LiteLLM Database
+
+```python
+import litellm
+
+# Create skill stored in LiteLLM DB
+skill = litellm.create_skill(
+ display_title="Data Analysis Skill",
+ files=[open("data-analysis.zip", "rb")],
+ custom_llm_provider="litellm_proxy", # Store in LiteLLM DB
+)
+
+print(f"Created skill: {skill.id}") # skill_abc123
+```
+
+### Use Skill with Any Provider
+
+```python
+import litellm
+
+# Use LiteLLM-stored skill with Bedrock
+response = litellm.completion(
+ model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+ messages=[{"role": "user", "content": "Analyze this data..."}],
+ container={
+ "skills": [
+ {"type": "custom", "skill_id": "litellm:skill_abc123"} # litellm: prefix
+ ]
+ }
+)
+```
+
+## How Skill Resolution Works
+
+### Step 1: Request with Skills
+
+```python
+{
+ "model": "bedrock/claude-3-sonnet",
+ "messages": [{"role": "user", "content": "Help me analyze data"}],
+ "container": {
+ "skills": [
+ {"type": "custom", "skill_id": "litellm:skill_abc123"}
+ ]
+ }
+}
+```
+
+### Step 2: SkillsInjectionHook Processing
+
+The hook (`litellm/proxy/hooks/litellm_skills/main.py`) intercepts the request:
+
+1. **Detects `litellm:` prefix** → Fetches skill from database
+2. **Checks model provider** → Bedrock is not Anthropic
+3. **Extracts SKILL.md** from stored ZIP file
+4. **Converts skill to tool** + **Injects content into system prompt**
+
+### Step 3: Transformed Request
+
+```python
+{
+ "model": "bedrock/claude-3-sonnet",
+ "messages": [
+ {
+ "role": "system",
+ "content": """
+---
+
+# Available Skills
+
+## Skill: Data Analysis Skill
+
+# Data Analysis Skill
+
+This skill helps with data analysis tasks...
+
+## Usage
+When the user asks about data analysis...
+"""
+ },
+ {"role": "user", "content": "Help me analyze data"}
+ ],
+ "tools": [
+ {
+ "type": "function",
+ "function": {
+ "name": "skill_abc123",
+ "description": "This skill helps with data analysis tasks...",
+ "parameters": {"type": "object", "properties": {}, "required": []}
+ }
+ }
+ ]
+ # container is removed for non-Anthropic providers
+}
+```
+
+## Database Schema
+
+Skills are stored in `LiteLLM_SkillsTable`:
+
+```prisma
+model LiteLLM_SkillsTable {
+ skill_id String @id @default(uuid())
+ display_title String?
+ description String?
+ instructions String?
+ source String @default("custom")
+ latest_version String?
+ metadata Json? @default("{}")
+ file_content Bytes? // ZIP file binary content
+ file_name String? // Original filename
+ file_type String? // MIME type
+ created_at DateTime @default(now())
+ created_by String?
+ updated_at DateTime @default(now()) @updatedAt
+ updated_by String?
+}
+```
+
+## Routing Summary
+
+| Scenario | custom_llm_provider | skill_id Format | Behavior |
+|----------|---------------------|-----------------|----------|
+| Create skill on Anthropic | `anthropic` | N/A | Forward to Anthropic API |
+| Create skill in LiteLLM DB | `litellm_proxy` | N/A | Store in database |
+| Use Anthropic native skill | N/A | `skill_xyz` | Pass to Anthropic container.skills |
+| Use LiteLLM skill on Anthropic | N/A | `litellm:skill_abc` | Convert to tools |
+| Use LiteLLM skill on Bedrock/OpenAI | N/A | `litellm:skill_abc` | Convert to tools + inject SKILL.md |
+
+## Testing
+
+Run the tests:
+
+```bash
+pytest tests/proxy_unit_tests/test_skills_db.py -v
+```
+
+Tests cover:
+- Creating skills with file content
+- Listing and retrieving skills
+- Deleting skills
+- Hook resolution with ZIP file extraction
+- System prompt injection for non-Anthropic models
+
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..5fb29e96bb9abd89697f351f3832287a2e36e446
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/__init__.py
@@ -0,0 +1,54 @@
+"""
+LiteLLM Proxy Skills - Database-backed skills storage and execution
+
+This module provides:
+- Database-backed skills storage (alternative to Anthropic's cloud-based skills API)
+- Skill content extraction and prompt injection
+- Sandboxed code execution for skills
+- Automatic code execution handler
+
+Main components:
+- handler.py: LiteLLMSkillsHandler - database CRUD operations
+- transformation.py: LiteLLMSkillsTransformationHandler - SDK transformation layer
+- prompt_injection.py: SkillPromptInjectionHandler - SKILL.md extraction and injection
+- sandbox_executor.py: SkillsSandboxExecutor - Docker sandbox execution
+- code_execution.py: CodeExecutionHandler - automatic agentic loop
+"""
+
+from litellm.llms.litellm_proxy.skills.code_execution import (
+ LITELLM_CODE_EXECUTION_TOOL,
+ CodeExecutionHandler,
+ LiteLLMInternalTools,
+ add_code_execution_tool,
+ code_execution_handler,
+ get_litellm_code_execution_tool,
+ has_code_execution_tool,
+)
+from litellm.llms.litellm_proxy.skills.constants import (
+ DEFAULT_MAX_ITERATIONS,
+ DEFAULT_SANDBOX_TIMEOUT,
+)
+from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
+from litellm.llms.litellm_proxy.skills.prompt_injection import (
+ SkillPromptInjectionHandler,
+)
+from litellm.llms.litellm_proxy.skills.sandbox_executor import SkillsSandboxExecutor
+from litellm.llms.litellm_proxy.skills.transformation import (
+ LiteLLMSkillsTransformationHandler,
+)
+
+__all__ = [
+ "LiteLLMSkillsHandler",
+ "LiteLLMSkillsTransformationHandler",
+ "SkillPromptInjectionHandler",
+ "SkillsSandboxExecutor",
+ "CodeExecutionHandler",
+ "LiteLLMInternalTools",
+ "LITELLM_CODE_EXECUTION_TOOL",
+ "get_litellm_code_execution_tool",
+ "code_execution_handler",
+ "has_code_execution_tool",
+ "add_code_execution_tool",
+ "DEFAULT_MAX_ITERATIONS",
+ "DEFAULT_SANDBOX_TIMEOUT",
+]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/code_execution.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/code_execution.py
new file mode 100644
index 0000000000000000000000000000000000000000..2b567f037604f00c50481cbc21228702a6a9cdc4
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/code_execution.py
@@ -0,0 +1,317 @@
+"""
+Automatic Code Execution Handler for LiteLLM Skills
+
+When `litellm_code_execution` tool is present, this handler automatically:
+1. Makes the LLM call
+2. Executes any code the model generates
+3. Continues the conversation with results
+4. Returns final response with generated files inline (base64)
+
+This mimics Anthropic's behavior where code execution happens automatically.
+Generated files are returned directly in the response - no separate storage needed.
+"""
+
+import base64
+import json
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from litellm._logging import verbose_logger
+
+
+class LiteLLMInternalTools(str, Enum):
+ """
+ Enum for internal LiteLLM tools that are injected into requests.
+
+ These tools are handled automatically by LiteLLM hooks and are not
+ passed to the underlying LLM provider directly.
+ """
+
+ CODE_EXECUTION = "litellm_code_execution"
+
+
+def get_litellm_code_execution_tool() -> Dict[str, Any]:
+ """
+ Returns the litellm_code_execution tool definition in OpenAI format.
+
+ This tool enables automatic code execution in a sandboxed environment
+ when skills include executable Python code.
+ """
+ return {
+ "type": "function",
+ "function": {
+ "name": LiteLLMInternalTools.CODE_EXECUTION.value,
+ "description": "Execute Python code in a sandboxed environment. Use this to run code that generates files, processes data, or performs computations. Generated files will be returned directly.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "code": {"type": "string", "description": "Python code to execute"}
+ },
+ "required": ["code"],
+ },
+ },
+ }
+
+
+def get_litellm_code_execution_tool_anthropic() -> Dict[str, Any]:
+ """
+ Returns the litellm_code_execution tool definition in Anthropic/messages API format.
+
+ This tool enables automatic code execution in a sandboxed environment
+ when skills include executable Python code.
+ """
+ return {
+ "name": LiteLLMInternalTools.CODE_EXECUTION.value,
+ "description": "Execute Python code in a sandboxed environment. Use this to run code that generates files, processes data, or performs computations. Generated files will be returned directly.",
+ "input_schema": {
+ "type": "object",
+ "properties": {
+ "code": {"type": "string", "description": "Python code to execute"}
+ },
+ "required": ["code"],
+ },
+ }
+
+
+# Singleton tool definition for backwards compatibility
+LITELLM_CODE_EXECUTION_TOOL = get_litellm_code_execution_tool()
+
+
+class CodeExecutionHandler:
+ """
+ Handles automatic code execution for LiteLLM skills.
+
+ When enabled, this handler intercepts LLM responses with code execution
+ tool calls, executes them in a sandbox, and continues the conversation
+ automatically until completion.
+ """
+
+ def __init__(
+ self,
+ max_iterations: Optional[int] = None,
+ sandbox_timeout: Optional[int] = None,
+ ):
+ from litellm.llms.litellm_proxy.skills.constants import (
+ DEFAULT_MAX_ITERATIONS,
+ DEFAULT_SANDBOX_TIMEOUT,
+ )
+
+ self.max_iterations = max_iterations or DEFAULT_MAX_ITERATIONS
+ self.sandbox_timeout = sandbox_timeout or DEFAULT_SANDBOX_TIMEOUT
+
+ async def execute_with_code_execution(
+ self,
+ model: str,
+ messages: List[Dict],
+ tools: List[Dict],
+ skill_files: Dict[str, bytes],
+ skill_id: Optional[str] = None,
+ **kwargs,
+ ) -> Dict[str, Any]:
+ """
+ Execute an LLM call with automatic code execution handling.
+
+ This method:
+ 1. Makes the initial LLM call
+ 2. If model calls litellm_code_execution, executes the code
+ 3. Continues conversation with results
+ 4. Repeats until model stops calling tools
+ 5. Returns final response with generated files inline
+
+ Args:
+ model: Model to use
+ messages: Initial messages
+ tools: Tools including litellm_code_execution
+ skill_files: Dict of skill files for execution
+ skill_id: Optional skill ID for tracking
+ **kwargs: Additional args for litellm.acompletion
+
+ Returns:
+ Dict with:
+ - response: Final LLM response
+ - files: List of generated files with content (base64)
+ - execution_results: List of code execution results
+ """
+ import litellm
+ from litellm.llms.litellm_proxy.skills.sandbox_executor import (
+ SkillsSandboxExecutor,
+ )
+
+ current_messages = list(messages)
+ generated_files: List[Dict[str, Any]] = [] # Files returned directly
+ execution_results: List[Dict] = []
+
+ executor = SkillsSandboxExecutor(timeout=self.sandbox_timeout)
+ response: Any = None # Initialize to avoid possibly unbound error
+
+ for iteration in range(self.max_iterations):
+ verbose_logger.debug(
+ f"CodeExecutionHandler: Iteration {iteration + 1}/{self.max_iterations}"
+ )
+
+ # Make LLM call
+ response = await litellm.acompletion(
+ model=model,
+ messages=current_messages,
+ tools=tools,
+ **kwargs,
+ )
+
+ assistant_message = response.choices[0].message # type: ignore
+ stop_reason = response.choices[0].finish_reason # type: ignore
+
+ # Build assistant message for conversation history
+ assistant_msg_dict: Dict[str, Any] = {
+ "role": "assistant",
+ "content": assistant_message.content,
+ }
+ if assistant_message.tool_calls:
+ assistant_msg_dict["tool_calls"] = [
+ {
+ "id": tc.id,
+ "type": "function",
+ "function": {
+ "name": tc.function.name,
+ "arguments": tc.function.arguments,
+ },
+ }
+ for tc in assistant_message.tool_calls
+ ]
+ current_messages.append(assistant_msg_dict)
+
+ # Check if we're done (no tool calls or not tool_calls finish reason)
+ if stop_reason != "tool_calls" or not assistant_message.tool_calls:
+ verbose_logger.debug(
+ f"CodeExecutionHandler: Completed after {iteration + 1} iterations"
+ )
+ return {
+ "response": response,
+ "files": generated_files, # Files returned directly with base64 content
+ "execution_results": execution_results,
+ "messages": current_messages,
+ }
+
+ # Handle tool calls
+ for tool_call in assistant_message.tool_calls:
+ tool_name = tool_call.function.name
+
+ if tool_name == LiteLLMInternalTools.CODE_EXECUTION.value:
+ # Execute code in sandbox
+ try:
+ args = json.loads(tool_call.function.arguments)
+ code = args.get("code", "")
+
+ verbose_logger.debug(
+ f"CodeExecutionHandler: Executing code ({len(code)} chars)"
+ )
+
+ exec_result = executor.execute(
+ code=code,
+ skill_files=skill_files,
+ )
+
+ verbose_logger.debug(
+ f"CodeExecutionHandler: Execution result: {exec_result}"
+ )
+
+ execution_results.append(
+ {
+ "iteration": iteration,
+ "success": exec_result["success"],
+ "output": exec_result["output"],
+ "error": exec_result["error"],
+ "files": [f["name"] for f in exec_result["files"]],
+ }
+ )
+
+ # Build tool result content
+ tool_result = exec_result["output"] or ""
+
+ # Collect generated files (returned directly, no storage)
+ if exec_result["files"]:
+ tool_result += "\n\nGenerated files:"
+ for f in exec_result["files"]:
+ file_content = base64.b64decode(f["content_base64"])
+ # Add to generated files list (returned in response)
+ generated_files.append(
+ {
+ "name": f["name"],
+ "mime_type": f["mime_type"],
+ "content_base64": f["content_base64"],
+ "size": len(file_content),
+ }
+ )
+ tool_result += (
+ f"\n- {f['name']} ({len(file_content)} bytes)"
+ )
+
+ verbose_logger.debug(
+ f"CodeExecutionHandler: Generated file {f['name']} ({len(file_content)} bytes)"
+ )
+
+ if exec_result["error"]:
+ tool_result += f"\n\nError:\n{exec_result['error']}"
+
+ except Exception as e:
+ tool_result = f"Code execution failed: {str(e)}"
+ execution_results.append(
+ {
+ "iteration": iteration,
+ "success": False,
+ "error": str(e),
+ }
+ )
+
+ # Add tool result to messages
+ current_messages.append(
+ {
+ "role": "tool",
+ "tool_call_id": tool_call.id,
+ "content": tool_result,
+ }
+ )
+ else:
+ # Non-code-execution tool - pass through
+ # In a full implementation, this would call other tool handlers
+ current_messages.append(
+ {
+ "role": "tool",
+ "tool_call_id": tool_call.id,
+ "content": f"Tool '{tool_name}' not handled by code execution handler",
+ }
+ )
+
+ # Max iterations reached
+ verbose_logger.warning(
+ f"CodeExecutionHandler: Max iterations ({self.max_iterations}) reached"
+ )
+ return {
+ "response": response,
+ "files": generated_files,
+ "execution_results": execution_results,
+ "messages": current_messages,
+ "max_iterations_reached": True,
+ }
+
+
+def has_code_execution_tool(tools: Optional[List[Dict]]) -> bool:
+ """Check if litellm_code_execution tool is in the tools list."""
+ if not tools:
+ return False
+ for tool in tools:
+ func = tool.get("function", {})
+ if func.get("name") == LiteLLMInternalTools.CODE_EXECUTION.value:
+ return True
+ return False
+
+
+def add_code_execution_tool(tools: Optional[List[Dict]]) -> List[Dict]:
+ """Add litellm_code_execution tool if not already present."""
+ tools = tools or []
+ if not has_code_execution_tool(tools):
+ tools.append(LITELLM_CODE_EXECUTION_TOOL)
+ return tools
+
+
+# Global handler instance
+code_execution_handler = CodeExecutionHandler()
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/constants.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/constants.py
new file mode 100644
index 0000000000000000000000000000000000000000..a8c2697fceebecb0dc2b524668d36790797930e3
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/constants.py
@@ -0,0 +1,12 @@
+"""
+Constants for LiteLLM Skills
+
+Centralized constants for skills processing, code execution, and sandbox configuration.
+"""
+
+# Code execution loop settings
+DEFAULT_MAX_ITERATIONS: int = 10
+"""Maximum number of iterations for the automatic code execution loop."""
+
+DEFAULT_SANDBOX_TIMEOUT: int = 120
+"""Default timeout in seconds for sandbox code execution."""
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..8e5070c2724ee2de73b515d887673391d6a14af7
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/handler.py
@@ -0,0 +1,219 @@
+"""
+Handler for LiteLLM database-backed skills operations.
+
+This module contains the actual database operations for skills CRUD.
+Used by the transformation layer and skills injection hook.
+"""
+
+import uuid
+from typing import Any, Dict, List, Optional
+
+from litellm._logging import verbose_logger
+from litellm.proxy._types import LiteLLM_SkillsTable, NewSkillRequest
+
+
+def _prisma_skill_to_litellm(prisma_skill) -> LiteLLM_SkillsTable:
+ """
+ Convert a Prisma skill record to LiteLLM_SkillsTable.
+
+ Handles Base64 decoding of file_content field.
+ """
+ import base64
+
+ data = prisma_skill.model_dump()
+
+ # Decode Base64 file_content back to bytes
+ # model_dump() converts Base64 field to base64-encoded string
+ if data.get("file_content") is not None:
+ if isinstance(data["file_content"], str):
+ data["file_content"] = base64.b64decode(data["file_content"])
+ elif isinstance(data["file_content"], bytes):
+ # Already bytes, no conversion needed
+ pass
+
+ return LiteLLM_SkillsTable(**data)
+
+
+class LiteLLMSkillsHandler:
+ """
+ Handler for LiteLLM database-backed skills operations.
+
+ This class provides static methods for CRUD operations on skills
+ stored in the LiteLLM proxy database (LiteLLM_SkillsTable).
+ """
+
+ @staticmethod
+ async def _get_prisma_client():
+ """Get the prisma client from proxy server."""
+ from litellm.proxy.proxy_server import prisma_client
+
+ if prisma_client is None:
+ raise ValueError(
+ "Prisma client is not initialized. "
+ "Database connection required for LiteLLM skills."
+ )
+ return prisma_client
+
+ @staticmethod
+ async def create_skill(
+ data: NewSkillRequest,
+ user_id: Optional[str] = None,
+ ) -> LiteLLM_SkillsTable:
+ """
+ Create a new skill in the LiteLLM database.
+
+ Args:
+ data: NewSkillRequest with skill details
+ user_id: Optional user ID for tracking
+
+ Returns:
+ LiteLLM_SkillsTable record
+ """
+ prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
+
+ skill_id = f"litellm_skill_{uuid.uuid4()}"
+
+ skill_data: Dict[str, Any] = {
+ "skill_id": skill_id,
+ "display_title": data.display_title,
+ "description": data.description,
+ "instructions": data.instructions,
+ "source": "custom",
+ "created_by": user_id,
+ "updated_by": user_id,
+ }
+
+ # Handle metadata
+ if data.metadata is not None:
+ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+
+ skill_data["metadata"] = safe_dumps(data.metadata)
+
+ # Handle file content - wrap bytes in Base64 for Prisma
+ if data.file_content is not None:
+ from prisma.fields import Base64
+
+ skill_data["file_content"] = Base64.encode(data.file_content)
+ if data.file_name is not None:
+ skill_data["file_name"] = data.file_name
+ if data.file_type is not None:
+ skill_data["file_type"] = data.file_type
+
+ verbose_logger.debug(
+ f"LiteLLMSkillsHandler: Creating skill {skill_id} with title={data.display_title}"
+ )
+
+ new_skill = await prisma_client.db.litellm_skillstable.create(data=skill_data)
+
+ return _prisma_skill_to_litellm(new_skill)
+
+ @staticmethod
+ async def list_skills(
+ limit: int = 20,
+ offset: int = 0,
+ ) -> List[LiteLLM_SkillsTable]:
+ """
+ List skills from the LiteLLM database.
+
+ Args:
+ limit: Maximum number of skills to return
+ offset: Number of skills to skip
+
+ Returns:
+ List of LiteLLM_SkillsTable records
+ """
+ prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
+
+ verbose_logger.debug(
+ f"LiteLLMSkillsHandler: Listing skills with limit={limit}, offset={offset}"
+ )
+
+ skills = await prisma_client.db.litellm_skillstable.find_many(
+ take=limit,
+ skip=offset,
+ order={"created_at": "desc"},
+ )
+
+ return [_prisma_skill_to_litellm(s) for s in skills]
+
+ @staticmethod
+ async def get_skill(skill_id: str) -> LiteLLM_SkillsTable:
+ """
+ Get a skill by ID from the LiteLLM database.
+
+ Args:
+ skill_id: The skill ID to retrieve
+
+ Returns:
+ LiteLLM_SkillsTable record
+
+ Raises:
+ ValueError: If skill not found
+ """
+ prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
+
+ verbose_logger.debug(f"LiteLLMSkillsHandler: Getting skill {skill_id}")
+
+ skill = await prisma_client.db.litellm_skillstable.find_unique(
+ where={"skill_id": skill_id}
+ )
+
+ if skill is None:
+ raise ValueError(f"Skill not found: {skill_id}")
+
+ return _prisma_skill_to_litellm(skill)
+
+ @staticmethod
+ async def delete_skill(skill_id: str) -> Dict[str, str]:
+ """
+ Delete a skill by ID from the LiteLLM database.
+
+ Args:
+ skill_id: The skill ID to delete
+
+ Returns:
+ Dict with id and type of deleted skill
+
+ Raises:
+ ValueError: If skill not found
+ """
+ prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
+
+ verbose_logger.debug(f"LiteLLMSkillsHandler: Deleting skill {skill_id}")
+
+ # Check if skill exists
+ skill = await prisma_client.db.litellm_skillstable.find_unique(
+ where={"skill_id": skill_id}
+ )
+
+ if skill is None:
+ raise ValueError(f"Skill not found: {skill_id}")
+
+ # Delete the skill
+ await prisma_client.db.litellm_skillstable.delete(where={"skill_id": skill_id})
+
+ return {"id": skill_id, "type": "skill_deleted"}
+
+ @staticmethod
+ async def fetch_skill_from_db(skill_id: str) -> Optional[LiteLLM_SkillsTable]:
+ """
+ Fetch a skill from the database (used by skills injection hook).
+
+ This is a convenience method that returns None instead of raising
+ an exception if the skill is not found.
+
+ Args:
+ skill_id: The skill ID to fetch
+
+ Returns:
+ LiteLLM_SkillsTable or None if not found
+ """
+ try:
+ return await LiteLLMSkillsHandler.get_skill(skill_id)
+ except ValueError:
+ return None
+ except Exception as e:
+ verbose_logger.warning(
+ f"LiteLLMSkillsHandler: Error fetching skill {skill_id}: {e}"
+ )
+ return None
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/prompt_injection.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/prompt_injection.py
new file mode 100644
index 0000000000000000000000000000000000000000..2b86f74122bf0b7457c2d9cb38cb056999012615
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/prompt_injection.py
@@ -0,0 +1,308 @@
+"""
+Prompt Injection Handler for LiteLLM Skills
+
+Handles extraction of skill content (SKILL.md) from stored ZIP files
+and injection into the system prompt for non-Anthropic models.
+"""
+
+import zipfile
+from io import BytesIO
+from typing import Any, Dict, List, Optional
+
+from litellm._logging import verbose_logger
+from litellm.proxy._types import LiteLLM_SkillsTable
+
+
+class SkillPromptInjectionHandler:
+ """
+ Handles skill content extraction and system prompt injection.
+
+ Responsibilities:
+ - Extract SKILL.md content from skill ZIP files
+ - Extract ALL files from ZIP for code execution
+ - Inject skill content into system message
+ - Create execute_code tool definition
+ """
+
+ def extract_skill_content(self, skill: LiteLLM_SkillsTable) -> Optional[str]:
+ """
+ Extract skill content from the stored zip file.
+
+ Looks for SKILL.md or README.md in the zip and returns its content.
+ This content describes the skill's capabilities and instructions.
+
+ Args:
+ skill: The skill from LiteLLM database
+
+ Returns:
+ The skill content as a string, or None if not available
+ """
+ if not skill.file_content:
+ return skill.instructions
+
+ try:
+ zip_buffer = BytesIO(skill.file_content)
+ with zipfile.ZipFile(zip_buffer, "r") as zf:
+ # Look for SKILL.md first
+ for name in zf.namelist():
+ if name.endswith("SKILL.md"):
+ content = zf.read(name).decode("utf-8")
+ if content:
+ return f"## Skill: {skill.display_title or skill.skill_id}\n\n{content}"
+
+ # Fall back to README.md
+ for name in zf.namelist():
+ if name.endswith("README.md"):
+ content = zf.read(name).decode("utf-8")
+ if content:
+ return f"## Skill: {skill.display_title or skill.skill_id}\n\n{content}"
+
+ # Fall back to any .md file
+ for name in zf.namelist():
+ if name.endswith(".md"):
+ content = zf.read(name).decode("utf-8")
+ if content:
+ return f"## Skill: {skill.display_title or skill.skill_id}\n\n{content}"
+ except Exception as e:
+ verbose_logger.warning(
+ f"SkillPromptInjectionHandler: Error extracting content from skill {skill.skill_id}: {e}"
+ )
+
+ return skill.instructions
+
+ def extract_all_files(self, skill: LiteLLM_SkillsTable) -> Dict[str, bytes]:
+ """
+ Extract ALL files from skill ZIP for code execution.
+
+ Returns a dict mapping file paths to their binary content.
+ The paths have the skill folder prefix removed (e.g., "slack-gif-creator/core/..." -> "core/...").
+
+ Args:
+ skill: The skill from LiteLLM database
+
+ Returns:
+ Dict mapping file paths to binary content
+ """
+ files: Dict[str, bytes] = {}
+
+ if not skill.file_content:
+ return files
+
+ try:
+ zip_buffer = BytesIO(skill.file_content)
+ with zipfile.ZipFile(zip_buffer, "r") as zf:
+ for name in zf.namelist():
+ # Skip directories
+ if name.endswith("/"):
+ continue
+
+ # Remove skill folder prefix (first path component)
+ parts = name.split("/")
+ if len(parts) > 1:
+ clean_path = "/".join(parts[1:])
+ else:
+ clean_path = name
+
+ if clean_path:
+ files[clean_path] = zf.read(name)
+ except Exception as e:
+ verbose_logger.warning(
+ f"SkillPromptInjectionHandler: Error extracting files from skill {skill.skill_id}: {e}"
+ )
+
+ return files
+
+ def inject_skill_content_to_messages(
+ self, data: dict, skill_contents: List[str], use_anthropic_format: bool = False
+ ) -> dict:
+ """
+ Inject skill content into the system prompt.
+
+ For Anthropic messages API (use_anthropic_format=True):
+ - Injects into top-level 'system' parameter (not in messages array)
+
+ For OpenAI-style APIs (use_anthropic_format=False):
+ - Injects into messages array with role="system"
+
+ Args:
+ data: The request data dict
+ skill_contents: List of skill content strings to inject
+ use_anthropic_format: If True, use top-level 'system' param for Anthropic
+
+ Returns:
+ Modified data dict with skill content in system prompt
+ """
+ if not skill_contents:
+ return data
+
+ # Build the skill injection text
+ skill_section = "\n\n---\n\n# Available Skills\n\n" + "\n\n---\n\n".join(
+ skill_contents
+ )
+
+ if use_anthropic_format:
+ # Anthropic messages API: use top-level 'system' parameter
+ current_system = data.get("system", "")
+ if current_system:
+ data["system"] = current_system + skill_section
+ else:
+ data["system"] = skill_section.strip()
+ return data
+
+ # OpenAI-style: inject into messages array
+ messages = data.get("messages", [])
+ if not messages:
+ return data
+
+ # Find or create system message
+ system_msg_idx = None
+ for i, msg in enumerate(messages):
+ if isinstance(msg, dict) and msg.get("role") == "system":
+ system_msg_idx = i
+ break
+
+ if system_msg_idx is not None:
+ # Append to existing system message
+ current_content = messages[system_msg_idx].get("content", "")
+ messages[system_msg_idx]["content"] = current_content + skill_section
+ else:
+ # Create new system message at the beginning
+ messages.insert(0, {"role": "system", "content": skill_section.strip()})
+
+ data["messages"] = messages
+ return data
+
+ def create_execute_code_tool(self, skill_modules: List[str]) -> Dict[str, Any]:
+ """
+ Create the execute_code tool definition.
+
+ This tool allows the model to execute Python code with access
+ to the skill's modules (e.g., 'from core.gif_builder import GIFBuilder').
+
+ Args:
+ skill_modules: List of available module paths (e.g., ["core/gif_builder.py"])
+
+ Returns:
+ OpenAI-style tool definition
+ """
+ # Format module list for description
+ module_examples = []
+ for mod in skill_modules[:5]: # Limit to 5 examples
+ if mod.endswith(".py"):
+ # Convert path to import: "core/gif_builder.py" -> "from core.gif_builder import ..."
+ import_path = mod.replace("/", ".").replace(".py", "")
+ module_examples.append(f"from {import_path} import ...")
+
+ module_hint = ""
+ if module_examples:
+ module_hint = f" Available modules: {', '.join(module_examples)}"
+
+ return {
+ "type": "function",
+ "function": {
+ "name": "execute_code",
+ "description": f"Execute Python code in a sandboxed environment. Generated files will be returned.{module_hint}",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "code": {
+ "type": "string",
+ "description": "Python code to execute. You can import skill modules and use standard libraries.",
+ }
+ },
+ "required": ["code"],
+ },
+ },
+ }
+
+ def convert_skill_to_tool(self, skill: LiteLLM_SkillsTable) -> Dict[str, Any]:
+ """
+ Convert a LiteLLM skill to an OpenAI-style tool.
+
+ The skill's instructions are used as the function description,
+ allowing the model to understand when and how to use the skill.
+
+ Args:
+ skill: The skill from LiteLLM database
+
+ Returns:
+ OpenAI-style tool definition
+ """
+ # Create a function name from skill_id (sanitize for function naming)
+ func_name = skill.skill_id.replace("-", "_").replace(" ", "_")
+
+ # Use instructions as description, fall back to description or title
+ description = (
+ skill.instructions
+ or skill.description
+ or skill.display_title
+ or f"Skill: {skill.skill_id}"
+ )
+
+ # Truncate description if too long (OpenAI has limits)
+ max_desc_length = 1024
+ if len(description) > max_desc_length:
+ description = description[: max_desc_length - 3] + "..."
+
+ tool: Dict[str, Any] = {
+ "type": "function",
+ "function": {
+ "name": func_name,
+ "description": description,
+ "parameters": {
+ "type": "object",
+ "properties": {},
+ "required": [],
+ },
+ },
+ }
+
+ # If skill has metadata with parameter definitions, use them
+ if skill.metadata and isinstance(skill.metadata, dict):
+ params = skill.metadata.get("parameters")
+ if params and isinstance(params, dict):
+ tool["function"]["parameters"] = params
+
+ return tool
+
+ def convert_skill_to_anthropic_tool(
+ self, skill: LiteLLM_SkillsTable
+ ) -> Dict[str, Any]:
+ """
+ Convert a LiteLLM skill to an Anthropic-style tool (messages API format).
+
+ Args:
+ skill: The skill from LiteLLM database
+
+ Returns:
+ Anthropic-style tool definition with name, description, input_schema
+ """
+ func_name = skill.skill_id.replace("-", "_").replace(" ", "_")
+
+ description = (
+ skill.instructions
+ or skill.description
+ or skill.display_title
+ or f"Skill: {skill.skill_id}"
+ )
+
+ max_desc_length = 1024
+ if len(description) > max_desc_length:
+ description = description[: max_desc_length - 3] + "..."
+
+ input_schema: Dict[str, Any] = {
+ "type": "object",
+ "properties": {},
+ "required": [],
+ }
+
+ if skill.metadata and isinstance(skill.metadata, dict):
+ params = skill.metadata.get("parameters")
+ if params and isinstance(params, dict):
+ input_schema = params
+
+ return {
+ "name": func_name,
+ "description": description,
+ "input_schema": input_schema,
+ }
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/sandbox_executor.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/sandbox_executor.py
new file mode 100644
index 0000000000000000000000000000000000000000..a5c0a539c96e7672a758922af10bfd40835acabf
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/sandbox_executor.py
@@ -0,0 +1,286 @@
+"""
+Sandbox Executor for LiteLLM Skills
+
+Executes skill code in a sandboxed environment using llm-sandbox.
+Supports Docker, Podman, and Kubernetes backends.
+"""
+
+import base64
+import os
+from typing import Any, Dict, List, Optional
+
+from litellm._logging import verbose_logger
+
+
+class SkillsSandboxExecutor:
+ """
+ Executes skill code in llm-sandbox Docker container.
+
+ Responsibilities:
+ - Create sandbox session with skill files
+ - Install requirements
+ - Execute model-generated code
+ - Collect generated files (GIFs, images, etc.)
+ """
+
+ def __init__(
+ self,
+ timeout: int = 60,
+ backend: str = "docker",
+ image: Optional[str] = None,
+ ):
+ """
+ Initialize the sandbox executor.
+
+ Args:
+ timeout: Maximum execution time in seconds
+ backend: Sandbox backend ("docker", "podman", "kubernetes")
+ image: Custom Docker image (default: uses llm-sandbox default)
+ """
+ self.timeout = timeout
+ self.backend = backend
+ self.image = image
+ self._session = None
+
+ def execute(
+ self,
+ code: str,
+ skill_files: Dict[str, bytes],
+ requirements: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """
+ Execute code with skill files in sandbox.
+
+ Args:
+ code: Python code to execute
+ skill_files: Dict mapping file paths to binary content
+ requirements: Optional requirements.txt content
+
+ Returns:
+ {
+ "success": bool,
+ "output": str,
+ "error": str (if failed),
+ "files": [{"name": str, "content_base64": str, "mime_type": str}]
+ }
+ """
+ try:
+ from llm_sandbox import SandboxSession
+ except ImportError:
+ verbose_logger.error(
+ "SkillsSandboxExecutor: llm-sandbox not installed. "
+ "Install with: pip install llm-sandbox"
+ )
+ return {
+ "success": False,
+ "output": "",
+ "error": "llm-sandbox not installed. Install with: pip install llm-sandbox",
+ "files": [],
+ }
+
+ try:
+ # Create sandbox session
+ session_kwargs: Dict[str, Any] = {
+ "lang": "python",
+ "verbose": False,
+ }
+
+ if self.image:
+ session_kwargs["image"] = self.image
+
+ with SandboxSession(**session_kwargs) as session:
+ # 1. Copy skill files into sandbox using copy_to_runtime
+ import tempfile
+
+ # Create a temp directory to stage files
+ with tempfile.TemporaryDirectory() as tmpdir:
+ for path, content in skill_files.items():
+ # Create the file in temp directory
+ local_path = os.path.join(tmpdir, path)
+ os.makedirs(os.path.dirname(local_path), exist_ok=True)
+ with open(local_path, "wb") as f:
+ f.write(content)
+
+ # Copy to sandbox
+ sandbox_path = f"/sandbox/{path}"
+ session.copy_to_runtime(local_path, sandbox_path)
+
+ verbose_logger.debug(
+ f"SkillsSandboxExecutor: Copied {len(skill_files)} files to sandbox"
+ )
+
+ # 2. Install requirements if present
+ req_packages = None
+ if requirements:
+ req_packages = requirements.strip().replace("\n", " ")
+ elif "requirements.txt" in skill_files:
+ req_content = skill_files["requirements.txt"].decode("utf-8")
+ req_packages = req_content.strip().replace("\n", " ")
+
+ if req_packages:
+ # Run pip install as code
+ pip_code = f"""
+import subprocess
+subprocess.run(['pip', 'install'] + '{req_packages}'.split(), check=True)
+"""
+ result = session.run(pip_code)
+ verbose_logger.debug(
+ "SkillsSandboxExecutor: Installed requirements"
+ )
+
+ # 3. Execute the code
+ # Wrap code to run from /sandbox directory
+ wrapped_code = f"""
+import os
+os.chdir('/sandbox')
+import sys
+sys.path.insert(0, '/sandbox')
+
+{code}
+"""
+ result = session.run(wrapped_code)
+
+ success = result.exit_code == 0
+ output = result.stdout or ""
+ error = result.stderr or ""
+
+ if success:
+ verbose_logger.debug(
+ "SkillsSandboxExecutor: Code execution succeeded"
+ )
+ else:
+ verbose_logger.debug(
+ f"SkillsSandboxExecutor: Code execution failed with exit code {result.exit_code}"
+ )
+ verbose_logger.debug(
+ f"SkillsSandboxExecutor: stderr: {error[:500] if error else 'No stderr'}"
+ )
+ verbose_logger.debug(
+ f"SkillsSandboxExecutor: stdout: {output[:500] if output else 'No stdout'}"
+ )
+
+ # 4. Collect generated files
+ generated_files = self._collect_generated_files(session, skill_files)
+
+ return {
+ "success": success,
+ "output": output,
+ "error": error,
+ "files": generated_files,
+ }
+
+ except Exception as e:
+ verbose_logger.error(f"SkillsSandboxExecutor: Execution failed: {e}")
+ return {
+ "success": False,
+ "output": "",
+ "error": str(e),
+ "files": [],
+ }
+
+ def _collect_generated_files(
+ self,
+ session: Any,
+ original_files: Dict[str, bytes],
+ ) -> List[Dict[str, Any]]:
+ """
+ Collect files generated during execution.
+
+ Looks for new files in /sandbox that weren't in the original skill files.
+ Focuses on common output types: GIF, PNG, JPG, PDF, CSV, etc.
+
+ Args:
+ session: The sandbox session
+ original_files: Original skill files (to exclude)
+
+ Returns:
+ List of generated files with base64 content
+ """
+ generated_files: List[Dict[str, Any]] = []
+
+ try:
+ import tempfile
+
+ # List files in /sandbox using Python code
+ list_code = """
+import os
+import json
+files = []
+for root, dirs, filenames in os.walk('/sandbox'):
+ for f in filenames:
+ if f.endswith(('.gif', '.png', '.jpg', '.jpeg', '.pdf', '.csv', '.json')):
+ files.append(os.path.join(root, f))
+print(json.dumps(files))
+"""
+ result = session.run(list_code)
+
+ if result.exit_code == 0 and result.stdout:
+ import json
+
+ try:
+ filepaths = json.loads(result.stdout.strip())
+ except json.JSONDecodeError:
+ filepaths = []
+
+ for filepath in filepaths:
+ if not filepath:
+ continue
+
+ # Get relative path
+ rel_path = filepath.replace("/sandbox/", "")
+
+ # Skip if it was an original file
+ if rel_path in original_files:
+ continue
+
+ # Copy file from sandbox using copy_from_runtime
+ with tempfile.NamedTemporaryFile(delete=False) as tmp:
+ tmp_path = tmp.name
+
+ try:
+ session.copy_from_runtime(filepath, tmp_path)
+
+ with open(tmp_path, "rb") as f:
+ content = f.read()
+
+ content_b64 = base64.b64encode(content).decode("utf-8")
+ generated_files.append(
+ {
+ "name": os.path.basename(filepath),
+ "path": rel_path,
+ "content_base64": content_b64,
+ "mime_type": self._get_mime_type(filepath),
+ }
+ )
+
+ verbose_logger.debug(
+ f"SkillsSandboxExecutor: Collected generated file: {rel_path}"
+ )
+ except Exception as e:
+ verbose_logger.warning(
+ f"SkillsSandboxExecutor: Error copying file {filepath}: {e}"
+ )
+ finally:
+ if os.path.exists(tmp_path):
+ os.unlink(tmp_path)
+
+ except Exception as e:
+ verbose_logger.warning(
+ f"SkillsSandboxExecutor: Error collecting generated files: {e}"
+ )
+
+ return generated_files
+
+ def _get_mime_type(self, filename: str) -> str:
+ """Get MIME type for a file based on extension."""
+ ext = filename.lower().split(".")[-1]
+ return {
+ "gif": "image/gif",
+ "png": "image/png",
+ "jpg": "image/jpeg",
+ "jpeg": "image/jpeg",
+ "pdf": "application/pdf",
+ "csv": "text/csv",
+ "json": "application/json",
+ "txt": "text/plain",
+ }.get(ext, "application/octet-stream")
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..4622bda4e8040fa79f8370d86842d85fe63c4f46
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/transformation.py
@@ -0,0 +1,345 @@
+"""
+Transformation handler for LiteLLM database-backed skills.
+
+This module provides the SDK-level transformation layer that converts
+API requests to database operations via LiteLLMSkillsHandler.
+
+Pattern follows litellm/llms/litellm_proxy/responses/transformation.py
+"""
+
+from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Union
+
+from litellm.types.llms.anthropic_skills import (
+ DeleteSkillResponse,
+ ListSkillsResponse,
+ Skill,
+)
+from litellm.types.utils import LlmProviders
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+
+class LiteLLMSkillsTransformationHandler:
+ """
+ Transformation handler for skills API requests to LiteLLM database operations.
+
+ This is used when custom_llm_provider="litellm_proxy" to store/retrieve skills
+ from the LiteLLM proxy database instead of calling an external API.
+ """
+
+ @property
+ def custom_llm_provider(self) -> str:
+ """Return the provider name for logging."""
+ return LlmProviders.LITELLM_PROXY.value
+
+ def create_skill_handler(
+ self,
+ display_title: Optional[str] = None,
+ description: Optional[str] = None,
+ instructions: Optional[str] = None,
+ files: Optional[List[Any]] = None,
+ file_content: Optional[bytes] = None,
+ file_name: Optional[str] = None,
+ file_type: Optional[str] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ user_id: Optional[str] = None,
+ _is_async: bool = False,
+ logging_obj: Optional["LiteLLMLoggingObj"] = None,
+ litellm_call_id: Optional[str] = None,
+ **kwargs,
+ ) -> Union[Skill, Coroutine[Any, Any, Skill]]:
+ """
+ Create a skill in LiteLLM database.
+
+ Args:
+ display_title: Display title for the skill
+ description: Description of the skill
+ instructions: Instructions/prompt for the skill
+ files: Files to upload - list of tuples (filename, content, content_type)
+ file_content: Binary content of skill files (alternative to files)
+ file_name: Original filename (alternative to files)
+ file_type: MIME type (alternative to files)
+ metadata: Additional metadata
+ user_id: User ID for tracking
+ _is_async: Whether to return a coroutine
+
+ Returns:
+ Skill object or coroutine that returns Skill
+ """
+ # Pre-call logging
+ if logging_obj:
+ logging_obj.update_from_kwargs(
+ kwargs=kwargs,
+ model=None,
+ optional_params={"display_title": display_title},
+ litellm_params={"litellm_call_id": litellm_call_id},
+ custom_llm_provider=self.custom_llm_provider,
+ )
+
+ # Extract file content from files parameter if provided
+ # files is a list of tuples: [(filename, content, content_type), ...]
+ if files and not file_content:
+ if isinstance(files, list) and len(files) > 0:
+ first_file = files[0]
+ if isinstance(first_file, tuple) and len(first_file) >= 2:
+ file_name = first_file[0]
+ file_content = first_file[1]
+ file_type = (
+ first_file[2] if len(first_file) > 2 else "application/zip"
+ )
+
+ if _is_async:
+ return self._async_create_skill(
+ display_title=display_title,
+ description=description,
+ instructions=instructions,
+ file_content=file_content,
+ file_name=file_name,
+ file_type=file_type,
+ metadata=metadata,
+ user_id=user_id,
+ )
+
+ import asyncio
+
+ return asyncio.get_event_loop().run_until_complete(
+ self._async_create_skill(
+ display_title=display_title,
+ description=description,
+ instructions=instructions,
+ file_content=file_content,
+ file_name=file_name,
+ file_type=file_type,
+ metadata=metadata,
+ user_id=user_id,
+ )
+ )
+
+ async def _async_create_skill(
+ self,
+ display_title: Optional[str] = None,
+ description: Optional[str] = None,
+ instructions: Optional[str] = None,
+ file_content: Optional[bytes] = None,
+ file_name: Optional[str] = None,
+ file_type: Optional[str] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ user_id: Optional[str] = None,
+ ) -> Skill:
+ """Async implementation of create_skill."""
+ # Lazy import to avoid SDK dependency on proxy
+ from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
+ from litellm.proxy._types import NewSkillRequest
+
+ skill_request = NewSkillRequest(
+ display_title=display_title,
+ description=description,
+ instructions=instructions,
+ file_content=file_content,
+ file_name=file_name,
+ file_type=file_type,
+ metadata=metadata,
+ )
+
+ db_skill = await LiteLLMSkillsHandler.create_skill(
+ data=skill_request,
+ user_id=user_id,
+ )
+
+ return self._db_skill_to_response(db_skill)
+
+ def list_skills_handler(
+ self,
+ limit: int = 20,
+ offset: int = 0,
+ _is_async: bool = False,
+ logging_obj: Optional["LiteLLMLoggingObj"] = None,
+ litellm_call_id: Optional[str] = None,
+ **kwargs,
+ ) -> Union[ListSkillsResponse, Coroutine[Any, Any, ListSkillsResponse]]:
+ """
+ List skills from LiteLLM database.
+
+ Args:
+ limit: Maximum number of skills to return
+ offset: Number of skills to skip
+ _is_async: Whether to return a coroutine
+ logging_obj: LiteLLM logging object
+ litellm_call_id: Call ID for logging
+
+ Returns:
+ ListSkillsResponse or coroutine that returns ListSkillsResponse
+ """
+ # Pre-call logging
+ if logging_obj:
+ logging_obj.update_from_kwargs(
+ kwargs=kwargs,
+ model=None,
+ optional_params={"limit": limit, "offset": offset},
+ litellm_params={"litellm_call_id": litellm_call_id},
+ custom_llm_provider=self.custom_llm_provider,
+ )
+
+ if _is_async:
+ return self._async_list_skills(limit=limit, offset=offset)
+
+ import asyncio
+
+ return asyncio.get_event_loop().run_until_complete(
+ self._async_list_skills(limit=limit, offset=offset)
+ )
+
+ async def _async_list_skills(
+ self,
+ limit: int = 20,
+ offset: int = 0,
+ ) -> ListSkillsResponse:
+ """Async implementation of list_skills."""
+ # Lazy import to avoid SDK dependency on proxy
+ from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
+
+ db_skills = await LiteLLMSkillsHandler.list_skills(
+ limit=limit,
+ offset=offset,
+ )
+
+ skills = [self._db_skill_to_response(s) for s in db_skills]
+ return ListSkillsResponse(
+ data=skills,
+ has_more=len(skills) >= limit,
+ next_page=None,
+ )
+
+ def get_skill_handler(
+ self,
+ skill_id: str,
+ _is_async: bool = False,
+ logging_obj: Optional["LiteLLMLoggingObj"] = None,
+ litellm_call_id: Optional[str] = None,
+ **kwargs,
+ ) -> Union[Skill, Coroutine[Any, Any, Skill]]:
+ """
+ Get a skill from LiteLLM database.
+
+ Args:
+ skill_id: The skill ID to retrieve
+ _is_async: Whether to return a coroutine
+ logging_obj: LiteLLM logging object
+ litellm_call_id: Call ID for logging
+
+ Returns:
+ Skill or coroutine that returns Skill
+ """
+ # Pre-call logging
+ if logging_obj:
+ logging_obj.update_from_kwargs(
+ kwargs=kwargs,
+ model=None,
+ optional_params={"skill_id": skill_id},
+ litellm_params={"litellm_call_id": litellm_call_id},
+ custom_llm_provider=self.custom_llm_provider,
+ )
+
+ if _is_async:
+ return self._async_get_skill(skill_id=skill_id)
+
+ import asyncio
+
+ return asyncio.get_event_loop().run_until_complete(
+ self._async_get_skill(skill_id=skill_id)
+ )
+
+ async def _async_get_skill(self, skill_id: str) -> Skill:
+ """Async implementation of get_skill."""
+ # Lazy import to avoid SDK dependency on proxy
+ from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
+
+ db_skill = await LiteLLMSkillsHandler.get_skill(skill_id=skill_id)
+ return self._db_skill_to_response(db_skill)
+
+ def delete_skill_handler(
+ self,
+ skill_id: str,
+ _is_async: bool = False,
+ logging_obj: Optional["LiteLLMLoggingObj"] = None,
+ litellm_call_id: Optional[str] = None,
+ **kwargs,
+ ) -> Union[DeleteSkillResponse, Coroutine[Any, Any, DeleteSkillResponse]]:
+ """
+ Delete a skill from LiteLLM database.
+
+ Args:
+ skill_id: The skill ID to delete
+ _is_async: Whether to return a coroutine
+ logging_obj: LiteLLM logging object
+ litellm_call_id: Call ID for logging
+
+ Returns:
+ DeleteSkillResponse or coroutine that returns DeleteSkillResponse
+ """
+ # Pre-call logging
+ if logging_obj:
+ logging_obj.update_from_kwargs(
+ kwargs=kwargs,
+ model=None,
+ optional_params={"skill_id": skill_id},
+ litellm_params={"litellm_call_id": litellm_call_id},
+ custom_llm_provider=self.custom_llm_provider,
+ )
+
+ if _is_async:
+ return self._async_delete_skill(skill_id=skill_id)
+
+ import asyncio
+
+ return asyncio.get_event_loop().run_until_complete(
+ self._async_delete_skill(skill_id=skill_id)
+ )
+
+ async def _async_delete_skill(self, skill_id: str) -> DeleteSkillResponse:
+ """Async implementation of delete_skill."""
+ # Lazy import to avoid SDK dependency on proxy
+ from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
+
+ result = await LiteLLMSkillsHandler.delete_skill(skill_id=skill_id)
+ return DeleteSkillResponse(
+ id=result["id"],
+ type=result.get("type", "skill_deleted"),
+ )
+
+ def _db_skill_to_response(self, db_skill: Any) -> Skill:
+ """
+ Convert a database skill record to Anthropic-compatible Skill response.
+
+ Args:
+ db_skill: LiteLLM_SkillsTable record
+
+ Returns:
+ Skill object
+ """
+ created_at = ""
+ updated_at = ""
+
+ if hasattr(db_skill, "created_at") and db_skill.created_at:
+ created_at = (
+ db_skill.created_at.isoformat()
+ if hasattr(db_skill.created_at, "isoformat")
+ else str(db_skill.created_at)
+ )
+ if hasattr(db_skill, "updated_at") and db_skill.updated_at:
+ updated_at = (
+ db_skill.updated_at.isoformat()
+ if hasattr(db_skill.updated_at, "isoformat")
+ else str(db_skill.updated_at)
+ )
+
+ return Skill(
+ id=db_skill.skill_id,
+ created_at=created_at,
+ updated_at=updated_at,
+ display_title=db_skill.display_title,
+ latest_version=db_skill.latest_version,
+ source=db_skill.source or "custom",
+ type="skill",
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/llamafile/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/llamafile/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..3387a0eb6aae259231ab01ad556fa879de637721
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/llamafile/chat/transformation.py
@@ -0,0 +1,45 @@
+from typing import Optional, Tuple
+
+from litellm.secret_managers.main import get_secret_str
+
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+
+
+class LlamafileChatConfig(OpenAIGPTConfig):
+ """LlamafileChatConfig is used to provide configuration for the LlamaFile's chat API."""
+
+ @staticmethod
+ def _resolve_api_key(api_key: Optional[str] = None) -> str:
+ """Attempt to ensure that the API key is set, preferring the user-provided key
+ over the secret manager key (``LLAMAFILE_API_KEY``).
+
+ If both are None, a fake API key is returned.
+ """
+ return (
+ api_key or get_secret_str("LLAMAFILE_API_KEY") or "fake-api-key"
+ ) # llamafile does not require an API key
+
+ @staticmethod
+ def _resolve_api_base(api_base: Optional[str] = None) -> Optional[str]:
+ """Attempt to ensure that the API base is set, preferring the user-provided key
+ over the secret manager key (``LLAMAFILE_API_BASE``).
+
+ If both are None, a default Llamafile server URL is returned.
+ See: https://github.com/Mozilla-Ocho/llamafile/blob/bd1bbe9aabb1ee12dbdcafa8936db443c571eb9d/README.md#L61
+ """
+ return api_base or get_secret_str("LLAMAFILE_API_BASE") or "http://127.0.0.1:8080/v1" # type: ignore
+
+ def _get_openai_compatible_provider_info(
+ self, api_base: Optional[str], api_key: Optional[str]
+ ) -> Tuple[Optional[str], Optional[str]]:
+ """Attempts to ensure that the API base and key are set, preferring user-provided values,
+ before falling back to secret manager values (``LLAMAFILE_API_BASE`` and ``LLAMAFILE_API_KEY``
+ respectively).
+
+ If an API key cannot be resolved via either method, a fake key is returned. Llamafile
+ does not require an API key, but the underlying OpenAI library may expect one anyway.
+ """
+ api_base = LlamafileChatConfig._resolve_api_base(api_base)
+ dynamic_api_key = LlamafileChatConfig._resolve_api_key(api_key)
+
+ return api_base, dynamic_api_key
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/lm_studio/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/lm_studio/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..64ed38467de8c2af5b6d52276ccd046c4756e20d
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/lm_studio/chat/transformation.py
@@ -0,0 +1,49 @@
+"""
+Translate from OpenAI's `/v1/chat/completions` to LM Studio's `/chat/completions`
+"""
+
+from typing import Optional, Tuple
+
+from litellm.secret_managers.main import get_secret_str
+
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+
+
+class LMStudioChatConfig(OpenAIGPTConfig):
+ def _get_openai_compatible_provider_info(
+ self, api_base: Optional[str], api_key: Optional[str]
+ ) -> Tuple[Optional[str], Optional[str]]:
+ api_base = api_base or get_secret_str("LM_STUDIO_API_BASE") # type: ignore
+ dynamic_api_key = (
+ api_key or get_secret_str("LM_STUDIO_API_KEY") or "fake-api-key"
+ ) # LM Studio does not require an api key, but OpenAI client requires non-None value
+ return api_base, dynamic_api_key
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ for param, value in list(non_default_params.items()):
+ if param == "response_format" and isinstance(value, dict):
+ if value.get("type") == "json_schema":
+ if "json_schema" not in value and "schema" in value:
+ optional_params["response_format"] = {
+ "type": "json_schema",
+ "json_schema": {"schema": value.get("schema")},
+ }
+ else:
+ optional_params["response_format"] = value
+ non_default_params.pop(param, None)
+ elif value.get("type") == "json_object":
+ optional_params["response_format"] = value
+ non_default_params.pop(param, None)
+
+ return super().map_openai_params(
+ non_default_params=non_default_params,
+ optional_params=optional_params,
+ model=model,
+ drop_params=drop_params,
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/lm_studio/embed/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/lm_studio/embed/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..1285550c30f92a6df6522f1ede13063bc8374d7d
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/lm_studio/embed/transformation.py
@@ -0,0 +1,50 @@
+"""
+Transformation logic from OpenAI /v1/embeddings format to LM Studio's `/v1/embeddings` format.
+
+Why separate file? Make it easy to see how transformation works
+
+Docs - https://lmstudio.ai/docs/basics/server
+"""
+
+import types
+from typing import List
+
+
+class LmStudioEmbeddingConfig:
+ """
+ Reference: https://lmstudio.ai/docs/basics/server
+ """
+
+ def __init__(
+ self,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return {
+ k: v
+ for k, v in cls.__dict__.items()
+ if not k.startswith("__")
+ and not isinstance(
+ v,
+ (
+ types.FunctionType,
+ types.BuiltinFunctionType,
+ classmethod,
+ staticmethod,
+ ),
+ )
+ and v is not None
+ }
+
+ def get_supported_openai_params(self) -> List[str]:
+ return []
+
+ def map_openai_params(
+ self, non_default_params: dict, optional_params: dict
+ ) -> dict:
+ return optional_params
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..03f1707d4466d331e892bc4f277def4b01be8e02
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/__init__.py
@@ -0,0 +1 @@
+# Manus provider implementation
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/files/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/files/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3659eef17c84212604264f4f5ca7ed3abb71c9ee
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/files/__init__.py
@@ -0,0 +1 @@
+# Manus Files API implementation
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/files/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/files/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..3381a5327e85aff05789e9f2d3f8f77e1a45a680
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/files/transformation.py
@@ -0,0 +1,434 @@
+"""
+Manus Files API implementation.
+
+Manus has an OpenAI-compatible Files API with some differences:
+- Uses API_KEY header instead of Authorization: Bearer
+- File upload is a two-step process:
+ 1. Create file record to get upload URL
+ 2. Upload file content to the upload URL
+
+Reference: https://open.manus.im/docs/openai-compatibility#file-management
+"""
+
+import time
+from typing import Any, Dict, List, Optional, Union
+
+import httpx
+from openai.types.file_deleted import FileDeleted
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.files.transformation import (
+ BaseFilesConfig,
+ LiteLLMLoggingObj,
+)
+from litellm.llms.openai.common_utils import OpenAIError
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.files import TwoStepFileUploadConfig, TwoStepFileUploadRequest
+from litellm.types.llms.openai import (
+ CreateFileRequest,
+ FileContentRequest,
+ HttpxBinaryResponseContent,
+ OpenAICreateFileRequestOptionalParams,
+ OpenAIFileObject,
+)
+from litellm.types.utils import LlmProviders
+
+MANUS_API_BASE = "https://api.manus.im"
+
+
+class ManusFilesConfig(BaseFilesConfig):
+ """
+ Configuration for Manus Files API.
+
+ Manus uses:
+ - API_KEY header for authentication (not Authorization: Bearer)
+ - Two-step file upload process
+ - Content-Type: application/json for all requests
+
+ Reference: https://open.manus.im/docs/openai-compatibility#file-management
+ """
+
+ def __init__(self):
+ pass
+
+ @property
+ def custom_llm_provider(self) -> LlmProviders:
+ return LlmProviders.MANUS
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages: list,
+ optional_params: dict,
+ litellm_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ """
+ Validate environment and set up headers for Manus API.
+
+ Manus uses API_KEY header instead of Authorization: Bearer.
+ For file uploads, don't set Content-Type - httpx will set it for multipart.
+ """
+ api_key = api_key or litellm.api_key or get_secret_str("MANUS_API_KEY")
+
+ if not api_key:
+ raise ValueError(
+ "Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter."
+ )
+
+ # Manus uses API_KEY header, not Authorization: Bearer
+ # Manus requires Content-Type: application/json for all requests (even GET)
+ headers.update(
+ {
+ "API_KEY": api_key,
+ "Content-Type": "application/json",
+ }
+ )
+ return headers
+
+ def get_supported_openai_params(
+ self, model: str
+ ) -> List[OpenAICreateFileRequestOptionalParams]:
+ """
+ Return supported OpenAI file creation parameters for Manus.
+ Manus supports the standard 'purpose' parameter.
+ """
+ return ["purpose"]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ """
+ Map OpenAI parameters to Manus-specific parameters.
+ Manus is OpenAI-compatible, so no special mapping needed.
+ """
+ return optional_params
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ api_key: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: dict,
+ stream: Optional[bool] = None,
+ ) -> str:
+ """
+ Get the complete URL for Manus Files API endpoint.
+
+ Returns:
+ str: The full URL for the Manus /v1/files endpoint
+ """
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("MANUS_API_BASE")
+ or MANUS_API_BASE
+ )
+
+ # Remove trailing slashes
+ api_base = api_base.rstrip("/")
+
+ # Manus API uses /v1/files endpoint
+ if api_base.endswith("/v1"):
+ return f"{api_base}/files"
+ return f"{api_base}/v1/files"
+
+ def get_error_class(
+ self,
+ error_message: str,
+ status_code: int,
+ headers: Union[dict, httpx.Headers],
+ ) -> BaseLLMException:
+ """
+ Return the appropriate error class for Manus API errors.
+ Uses OpenAIError since Manus is OpenAI-compatible.
+ """
+ return OpenAIError(
+ status_code=status_code,
+ message=error_message,
+ headers=headers,
+ )
+
+ def transform_create_file_request(
+ self,
+ model: str,
+ create_file_data: CreateFileRequest,
+ optional_params: dict,
+ litellm_params: dict,
+ ) -> TwoStepFileUploadConfig:
+ """
+ Transform OpenAI-style file creation request into Manus's two-step format.
+
+ Manus API spec (https://open.manus.im/docs/openai-compatibility#file-management):
+ 1. POST /v1/files with JSON {"filename": "..."} → returns {"id": "...", "upload_url": "..."}
+ 2. PUT to upload_url with raw file content
+ """
+ # Extract file data
+ file_data = create_file_data.get("file")
+ if file_data is None:
+ raise ValueError("File data is required")
+
+ extracted_data = extract_file_data(file_data)
+ filename = extracted_data["filename"] or f"file_{int(time.time())}"
+ content = extracted_data["content"]
+
+ # Get API base URL
+ api_base = self.get_complete_url(
+ api_base=litellm_params.get("api_base"),
+ api_key=litellm_params.get("api_key"),
+ model=model,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ )
+
+ # Get API key
+ api_key = (
+ litellm_params.get("api_key")
+ or litellm.api_key
+ or get_secret_str("MANUS_API_KEY")
+ )
+
+ if not api_key:
+ raise ValueError(
+ "Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter."
+ )
+
+ # Build typed two-step upload config
+ return TwoStepFileUploadConfig(
+ initial_request=TwoStepFileUploadRequest(
+ method="POST",
+ url=api_base,
+ headers={
+ "API_KEY": api_key,
+ "Content-Type": "application/json",
+ },
+ data={"filename": filename},
+ ),
+ upload_request=TwoStepFileUploadRequest(
+ method="PUT",
+ url="", # Will be populated from initial_request response
+ headers={},
+ data=content,
+ ),
+ upload_url_location="body",
+ upload_url_key="upload_url",
+ )
+
+ def transform_create_file_response(
+ self,
+ model: Optional[str],
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ litellm_params: dict,
+ ) -> OpenAIFileObject:
+ """
+ Transform Manus's file upload response into OpenAI-style FileObject.
+
+ For two-step uploads, the handler stores the initial response in litellm_params.
+ We need to return the file object from the initial POST, not the final PUT.
+
+ Manus initial response format:
+ {
+ "id": "file-abc123xyz",
+ "object": "file",
+ "filename": "document.pdf",
+ "status": "pending",
+ "upload_url": "https://...",
+ "upload_expires_at": "...",
+ "created_at": "..."
+ }
+ """
+ try:
+ # For two-step uploads, get the initial response from litellm_params
+ initial_response_data = litellm_params.get("initial_file_response")
+ if initial_response_data:
+ response_json = initial_response_data
+ else:
+ # Log raw response for debugging
+ verbose_logger.debug(f"Manus raw response text: {raw_response.text}")
+ response_json = raw_response.json()
+
+ verbose_logger.debug(f"Manus file response: {response_json}")
+
+ # Parse created_at timestamp
+ created_at_str = response_json.get("created_at", "")
+ if created_at_str:
+ try:
+ # Try parsing ISO format
+ created_at = int(
+ time.mktime(
+ time.strptime(
+ created_at_str.replace("Z", "+00:00")[:19],
+ "%Y-%m-%dT%H:%M:%S",
+ )
+ )
+ )
+ except (ValueError, TypeError):
+ created_at = int(time.time())
+ else:
+ created_at = int(time.time())
+
+ return OpenAIFileObject(
+ id=response_json.get("id", ""),
+ bytes=response_json.get("bytes", 0),
+ created_at=created_at,
+ filename=response_json.get("filename", ""),
+ object="file",
+ purpose=response_json.get("purpose", "assistants"),
+ status="uploaded", # After successful upload, status is uploaded
+ status_details=response_json.get("status_details"),
+ )
+ except Exception as e:
+ verbose_logger.exception(f"Error parsing Manus file response: {str(e)}")
+ raise ValueError(f"Error parsing Manus file response: {str(e)}")
+
+ def transform_retrieve_file_request(
+ self,
+ file_id: str,
+ optional_params: dict,
+ litellm_params: dict,
+ ) -> tuple[str, dict]:
+ """Get URL and params for retrieving a file."""
+ api_base = self.get_complete_url(
+ api_base=litellm_params.get("api_base"),
+ api_key=litellm_params.get("api_key"),
+ model="",
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ )
+ return f"{api_base}/{file_id}", {}
+
+ def transform_retrieve_file_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ litellm_params: dict,
+ ) -> OpenAIFileObject:
+ """Transform retrieve file response."""
+ return self.transform_create_file_response(
+ model=None,
+ raw_response=raw_response,
+ logging_obj=logging_obj,
+ litellm_params=litellm_params,
+ )
+
+ def transform_delete_file_request(
+ self,
+ file_id: str,
+ optional_params: dict,
+ litellm_params: dict,
+ ) -> tuple[str, dict]:
+ """Get URL and params for deleting a file."""
+ api_base = self.get_complete_url(
+ api_base=litellm_params.get("api_base"),
+ api_key=litellm_params.get("api_key"),
+ model="",
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ )
+ return f"{api_base}/{file_id}", {}
+
+ def transform_delete_file_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ litellm_params: dict,
+ ) -> FileDeleted:
+ """Transform delete file response."""
+ response_json = raw_response.json()
+ return FileDeleted(**response_json)
+
+ def transform_list_files_request(
+ self,
+ purpose: Optional[str],
+ optional_params: dict,
+ litellm_params: dict,
+ ) -> tuple[str, dict]:
+ """Get URL and params for listing files."""
+ api_base = self.get_complete_url(
+ api_base=litellm_params.get("api_base"),
+ api_key=litellm_params.get("api_key"),
+ model="",
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ )
+ params = {}
+ if purpose:
+ params["purpose"] = purpose
+ return api_base, params
+
+ def transform_list_files_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ litellm_params: dict,
+ ) -> List[OpenAIFileObject]:
+ """Transform list files response."""
+ response_json = raw_response.json()
+ files_data = response_json.get("data", [])
+ return [self._parse_file_dict(f) for f in files_data]
+
+ def _parse_file_dict(self, file_dict: Dict[str, Any]) -> OpenAIFileObject:
+ """Parse a file dict into OpenAIFileObject."""
+ created_at_str = file_dict.get("created_at", "")
+ if created_at_str:
+ try:
+ created_at = int(
+ time.mktime(
+ time.strptime(
+ created_at_str.replace("Z", "+00:00")[:19],
+ "%Y-%m-%dT%H:%M:%S",
+ )
+ )
+ )
+ except (ValueError, TypeError):
+ created_at = int(time.time())
+ else:
+ created_at = int(time.time())
+
+ return OpenAIFileObject(
+ id=file_dict.get("id", ""),
+ bytes=file_dict.get("bytes", 0),
+ created_at=created_at,
+ filename=file_dict.get("filename", ""),
+ object="file",
+ purpose=file_dict.get("purpose", "assistants"),
+ status=file_dict.get("status", "uploaded"),
+ status_details=file_dict.get("status_details"),
+ )
+
+ def transform_file_content_request(
+ self,
+ file_content_request: FileContentRequest,
+ optional_params: dict,
+ litellm_params: dict,
+ ) -> tuple[str, dict]:
+ """Get URL and params for retrieving file content."""
+ file_id = file_content_request.get("file_id")
+ api_base = self.get_complete_url(
+ api_base=litellm_params.get("api_base"),
+ api_key=litellm_params.get("api_key"),
+ model="",
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ )
+ return f"{api_base}/{file_id}/content", {}
+
+ def transform_file_content_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ litellm_params: dict,
+ ) -> HttpxBinaryResponseContent:
+ """Transform file content response."""
+ return HttpxBinaryResponseContent(response=raw_response)
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/responses/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/responses/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..7df60c923b7782bb87323a6941cb2def66bc846f
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/responses/__init__.py
@@ -0,0 +1 @@
+# Manus Responses API implementation
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/responses/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/responses/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..510c41304a86b39441ae91c9c5acb153669f5c72
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/responses/transformation.py
@@ -0,0 +1,353 @@
+import uuid
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
+
+import httpx
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import process_response_headers
+from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+ _safe_convert_created_field,
+)
+from litellm.llms.openai.common_utils import OpenAIError
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import (
+ ResponseAPIUsage,
+ ResponseInputParam,
+ ResponsesAPIResponse,
+)
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import LlmProviders
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+ LiteLLMLoggingObj = Any
+
+MANUS_API_BASE = "https://api.manus.im"
+
+
+class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
+ """
+ Configuration for Manus API's Responses API.
+
+ Manus API is OpenAI-compatible but has some differences:
+ - API key passed via `API_KEY` header (not `Authorization: Bearer`)
+ - Model format: `manus/{agent_profile}` (e.g., `manus/manus-1.6`)
+ - Requires `extra_body` with `task_mode: "agent"` and `agent_profile`
+
+ Reference: https://open.manus.im/docs/openai-compatibility
+ """
+
+ @property
+ def custom_llm_provider(self) -> LlmProviders:
+ return LlmProviders.MANUS
+
+ def should_fake_stream(
+ self,
+ model: Optional[str],
+ stream: Optional[bool],
+ custom_llm_provider: Optional[str] = None,
+ ) -> bool:
+ """
+ Manus API doesn't support real-time streaming.
+ It returns a task that runs asynchronously.
+ We fake streaming by converting the response into streaming events.
+ """
+ return stream is True
+
+ def _extract_agent_profile(self, model: str) -> str:
+ """
+ Extract agent profile from model name.
+
+ Model format: `manus/{agent_profile}`
+ Examples: `manus/manus-1.6`, `manus/manus-1.6-lite`, `manus/manus-1.6-max`
+
+ Returns:
+ str: The agent profile (e.g., "manus-1.6")
+ """
+ if "/" in model:
+ return model.split("/", 1)[1]
+ # If no slash, assume the model name itself is the agent profile
+ return model
+
+ def validate_environment(
+ self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
+ ) -> dict:
+ """
+ Validate environment and set up headers for Manus API.
+
+ Manus uses `API_KEY` header instead of `Authorization: Bearer`.
+ """
+ litellm_params = litellm_params or GenericLiteLLMParams()
+ api_key = (
+ litellm_params.api_key or litellm.api_key or get_secret_str("MANUS_API_KEY")
+ )
+
+ if not api_key:
+ raise ValueError(
+ "Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter."
+ )
+
+ # Manus uses API_KEY header, not Authorization: Bearer
+ # Content-Type is required for all requests (including GET)
+ headers.update(
+ {
+ "API_KEY": api_key,
+ "Content-Type": "application/json",
+ }
+ )
+ return headers
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ litellm_params: dict,
+ ) -> str:
+ """
+ Get the complete URL for Manus Responses API endpoint.
+
+ Returns:
+ str: The full URL for the Manus /v1/responses endpoint
+ """
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("MANUS_API_BASE")
+ or MANUS_API_BASE
+ )
+
+ # Remove trailing slashes
+ api_base = api_base.rstrip("/")
+
+ # Manus API uses /v1/responses endpoint (OpenAI-compatible)
+ if api_base.endswith("/v1"):
+ return f"{api_base}/responses"
+ return f"{api_base}/v1/responses"
+
+ def transform_responses_api_request(
+ self,
+ model: str,
+ input: Union[str, ResponseInputParam],
+ response_api_optional_request_params: Dict,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Dict:
+ """
+ Transform the request for Manus API.
+
+ Manus requires:
+ - `task_mode: "agent"` in the request body
+ - `agent_profile` extracted from model name in the request body
+ """
+ # First, get the base OpenAI request
+ base_request = super().transform_responses_api_request(
+ model=model,
+ input=input,
+ response_api_optional_request_params=response_api_optional_request_params,
+ litellm_params=litellm_params,
+ headers=headers,
+ )
+
+ # Extract agent profile from model name
+ agent_profile = self._extract_agent_profile(model=model)
+
+ # Add Manus-specific parameters directly to the request body
+ # These will be sent as part of the request
+ base_request["task_mode"] = "agent"
+ base_request["agent_profile"] = agent_profile
+
+ # Merge any existing extra_body into the request
+ extra_body = response_api_optional_request_params.get("extra_body", {}) or {}
+ if extra_body:
+ base_request.update(extra_body)
+
+ verbose_logger.debug(
+ f"Manus: Using agent_profile={agent_profile}, task_mode=agent"
+ )
+
+ return base_request
+
+ def transform_response_api_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ResponsesAPIResponse:
+ """
+ Transform Manus API response to OpenAI-compatible format.
+
+ Manus uses camelCase (createdAt) instead of snake_case (created_at).
+ """
+ try:
+ logging_obj.post_call(
+ original_response=raw_response.text,
+ additional_args={"complete_input_dict": {}},
+ )
+ raw_response_json = raw_response.json()
+
+ # Manus uses camelCase "createdAt" instead of snake_case "created_at"
+ if (
+ "createdAt" in raw_response_json
+ and "created_at" not in raw_response_json
+ ):
+ raw_response_json["created_at"] = _safe_convert_created_field(
+ raw_response_json["createdAt"]
+ )
+
+ # Ensure created_at is set
+ if "created_at" in raw_response_json:
+ raw_response_json["created_at"] = _safe_convert_created_field(
+ raw_response_json["created_at"]
+ )
+ except Exception:
+ raise OpenAIError(
+ message=raw_response.text, status_code=raw_response.status_code
+ )
+
+ raw_response_headers = dict(raw_response.headers)
+ processed_headers = process_response_headers(raw_response_headers)
+
+ # Ensure reasoning is an empty dict if not present, OpenAI SDK does not allow None
+ if (
+ "reasoning" not in raw_response_json
+ or raw_response_json.get("reasoning") is None
+ ):
+ raw_response_json["reasoning"] = {}
+
+ if "text" not in raw_response_json or raw_response_json.get("text") is None:
+ raw_response_json["text"] = {}
+
+ if "output" not in raw_response_json or raw_response_json.get("output") is None:
+ raw_response_json["output"] = []
+
+ # Ensure usage is present with default values if not provided
+ if "usage" not in raw_response_json or raw_response_json.get("usage") is None:
+ raw_response_json["usage"] = ResponseAPIUsage(
+ input_tokens=0,
+ output_tokens=0,
+ total_tokens=0,
+ )
+
+ # Ensure id is present - failed responses may not include it
+ if "id" not in raw_response_json or raw_response_json.get("id") is None:
+ # Generate a placeholder id for failed responses
+ # This allows the response object to be created even when the API doesn't return an id
+ raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
+
+ try:
+ response = ResponsesAPIResponse(**raw_response_json)
+ except Exception:
+ verbose_logger.debug(
+ f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct"
+ )
+ response = ResponsesAPIResponse.model_construct(**raw_response_json)
+
+ # Store processed headers in additional_headers so they get returned to the client
+ response._hidden_params["additional_headers"] = processed_headers
+ response._hidden_params["headers"] = raw_response_headers
+ return response
+
+ def supports_native_websocket(self) -> bool:
+ """Manus does not support native WebSocket for Responses API"""
+ return False
+
+ def transform_get_response_api_request(
+ self,
+ response_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """
+ Transform the get response API request into a URL and data.
+
+ Manus API follows OpenAI-compatible format:
+ - GET /v1/responses/{response_id}
+
+ Reference: https://open.manus.im/docs/openai-compatibility
+ """
+ url = f"{api_base}/{response_id}"
+ data: Dict = {}
+ return url, data
+
+ def transform_get_response_api_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ResponsesAPIResponse:
+ """
+ Transform Manus API GET response to OpenAI-compatible format.
+
+ Manus uses camelCase (createdAt) instead of snake_case (created_at).
+ Same transformation as transform_response_api_response.
+ """
+ try:
+ logging_obj.post_call(
+ original_response=raw_response.text,
+ additional_args={"complete_input_dict": {}},
+ )
+ raw_response_json = raw_response.json()
+
+ # Manus uses camelCase "createdAt" instead of snake_case "created_at"
+ if (
+ "createdAt" in raw_response_json
+ and "created_at" not in raw_response_json
+ ):
+ raw_response_json["created_at"] = _safe_convert_created_field(
+ raw_response_json["createdAt"]
+ )
+
+ # Ensure created_at is set
+ if "created_at" in raw_response_json:
+ raw_response_json["created_at"] = _safe_convert_created_field(
+ raw_response_json["created_at"]
+ )
+ except Exception:
+ raise OpenAIError(
+ message=raw_response.text, status_code=raw_response.status_code
+ )
+
+ raw_response_headers = dict(raw_response.headers)
+ processed_headers = process_response_headers(raw_response_headers)
+
+ # Ensure reasoning, text, output, and usage are present with defaults
+ if (
+ "reasoning" not in raw_response_json
+ or raw_response_json.get("reasoning") is None
+ ):
+ raw_response_json["reasoning"] = {}
+
+ if "text" not in raw_response_json or raw_response_json.get("text") is None:
+ raw_response_json["text"] = {}
+
+ if "output" not in raw_response_json or raw_response_json.get("output") is None:
+ raw_response_json["output"] = []
+
+ if "usage" not in raw_response_json or raw_response_json.get("usage") is None:
+ raw_response_json["usage"] = ResponseAPIUsage(
+ input_tokens=0,
+ output_tokens=0,
+ total_tokens=0,
+ )
+
+ # Ensure id is present - failed responses may not include it
+ if "id" not in raw_response_json or raw_response_json.get("id") is None:
+ # Generate a placeholder id for failed responses
+ raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
+
+ try:
+ response = ResponsesAPIResponse(**raw_response_json)
+ except Exception:
+ verbose_logger.debug(
+ f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct"
+ )
+ response = ResponsesAPIResponse.model_construct(**raw_response_json)
+
+ # Store processed headers in additional_headers so they get returned to the client
+ response._hidden_params["additional_headers"] = processed_headers
+ response._hidden_params["headers"] = raw_response_headers
+ return response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/maritalk.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/maritalk.py
new file mode 100644
index 0000000000000000000000000000000000000000..418d13b3448ada02f112f86a881926e35fcc9381
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/maritalk.py
@@ -0,0 +1,65 @@
+from typing import List, Optional, Union
+
+from httpx._models import Headers
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+
+
+class MaritalkError(BaseLLMException):
+ def __init__(
+ self,
+ status_code: int,
+ message: str,
+ headers: Optional[Union[dict, Headers]] = None,
+ ):
+ super().__init__(status_code=status_code, message=message, headers=headers)
+
+
+class MaritalkConfig(OpenAIGPTConfig):
+ def __init__(
+ self,
+ frequency_penalty: Optional[float] = None,
+ presence_penalty: Optional[float] = None,
+ top_p: Optional[float] = None,
+ top_k: Optional[int] = None,
+ temperature: Optional[float] = None,
+ max_tokens: Optional[int] = None,
+ n: Optional[int] = None,
+ stop: Optional[List[str]] = None,
+ stream: Optional[bool] = None,
+ stream_options: Optional[dict] = None,
+ tools: Optional[List[dict]] = None,
+ tool_choice: Optional[Union[str, dict]] = None,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return super().get_config()
+
+ def get_supported_openai_params(self, model: str) -> List:
+ return [
+ "frequency_penalty",
+ "presence_penalty",
+ "top_p",
+ "top_k",
+ "temperature",
+ "max_tokens",
+ "n",
+ "stop",
+ "stream",
+ "stream_options",
+ "tools",
+ "tool_choice",
+ ]
+
+ def get_error_class(
+ self, error_message: str, status_code: int, headers: Union[dict, Headers]
+ ) -> BaseLLMException:
+ return MaritalkError(
+ status_code=status_code, message=error_message, headers=headers
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/meta_llama/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/meta_llama/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..6c9b79005f56afeac8a9f471f67265628d3e9587
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/meta_llama/chat/transformation.py
@@ -0,0 +1,46 @@
+"""
+Support for Llama API's `https://api.llama.com/compat/v1` endpoint.
+
+Calls done in OpenAI/openai.py as Llama API is openai-compatible.
+
+Docs: https://llama.developer.meta.com/docs/features/compatibility/
+"""
+
+import warnings
+
+# Suppress Pydantic serialization warnings for Meta Llama responses
+warnings.filterwarnings("ignore", message="Pydantic serializer warnings")
+
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+
+
+class LlamaAPIConfig(OpenAIGPTConfig):
+ def get_supported_openai_params(self, model: str) -> list:
+ """
+ Llama API has limited support for OpenAI parameters
+
+ function_call, tools, and tool_choice are working
+ response_format: only json_schema is working
+ """
+ # Function calling and tool choice are now supported on Llama API
+ optional_params = super().get_supported_openai_params(model)
+ return optional_params
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ mapped_openai_params = super().map_openai_params(
+ non_default_params, optional_params, model, drop_params
+ )
+
+ # Only json_schema is working for response_format
+ if (
+ "response_format" in mapped_openai_params
+ and mapped_openai_params["response_format"].get("type") != "json_schema"
+ ):
+ mapped_openai_params.pop("response_format")
+ return mapped_openai_params
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/milvus/vector_stores/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/milvus/vector_stores/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..c20f5fa94b7a59ad263ab4ee77a0daaeed25ed8f
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/milvus/vector_stores/__init__.py
@@ -0,0 +1,3 @@
+from litellm.llms.milvus.vector_stores.transformation import MilvusVectorStoreConfig
+
+__all__ = ["MilvusVectorStoreConfig"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/milvus/vector_stores/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/milvus/vector_stores/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..fcf5d14db7cff2bcd7ffe72b772fa46f6b1a57d6
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/milvus/vector_stores/transformation.py
@@ -0,0 +1,281 @@
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+
+import httpx
+
+import litellm
+from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.vector_stores import (
+ BaseVectorStoreAuthCredentials,
+ VectorStoreCreateOptionalRequestParams,
+ VectorStoreCreateResponse,
+ VectorStoreIndexEndpoints,
+ VectorStoreResultContent,
+ VectorStoreSearchOptionalRequestParams,
+ VectorStoreSearchResponse,
+ VectorStoreSearchResult,
+)
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+ LiteLLMLoggingObj = Any
+
+MILVUS_OPTIONAL_PARAMS = {
+ "dbName",
+ "annsField",
+ "limit",
+ "filter",
+ "offset",
+ "groupingField",
+ "outputFields",
+ "searchParams",
+ "partitionNames",
+ "consistencyLevel",
+}
+
+
+class MilvusVectorStoreConfig(BaseVectorStoreConfig):
+ """
+ Configuration for Milvus Vector Store
+
+ This implementation uses the Azure AI Search API for vector store operations.
+ Supports vector search with embeddings generated via litellm.embeddings.
+ """
+
+ def __init__(self):
+ super().__init__()
+
+ def validate_environment(
+ self, headers: dict, litellm_params: Optional[GenericLiteLLMParams]
+ ) -> dict:
+ api_key: Optional[str] = None
+ if litellm_params is not None:
+ api_key = litellm_params.api_key or get_secret_str("MILVUS_API_KEY")
+
+ if not api_key:
+ raise ValueError(
+ "MILVUS_API_KEY is not set. Either set it in the litellm_params or set the MILVUS_API_KEY environment variable."
+ )
+
+ headers.update({"Authorization": f"Bearer {api_key}"})
+
+ return headers
+
+ def get_auth_credentials(
+ self, litellm_params: dict
+ ) -> BaseVectorStoreAuthCredentials:
+ api_key = litellm_params.get("api_key")
+ if not api_key:
+ raise ValueError(
+ "MILVUS_API_KEY is not set. Either set it in the litellm_params or set the MILVUS_API_KEY environment variable."
+ )
+ return {
+ "headers": {
+ "Authorization": f"Bearer {api_key}",
+ },
+ }
+
+ def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints:
+ return {
+ "read": [
+ ("POST", "/v2/vectordb/entities/search"),
+ ("POST", "/v2/vectordb/entities/get"),
+ ("POST", "/v2/vectordb/entities/query"),
+ ],
+ "write": [
+ ("POST", "/v2/vectordb/entities/upsert"),
+ ("POST", "/v2/vectordb/entities/insert"),
+ ],
+ }
+
+ def map_openai_params(
+ self, non_default_params: dict, optional_params: dict, drop_params: bool
+ ) -> dict:
+ for param, value in non_default_params.items():
+ if param in MILVUS_OPTIONAL_PARAMS:
+ optional_params[param] = value
+ return optional_params
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ litellm_params: dict,
+ ) -> str:
+ """
+ Get the base endpoint for Milvus API
+
+ Expected format: https://{milvus_api_base}.milvus.io
+ """
+ api_base = api_base or get_secret_str("MILVUS_API_BASE")
+
+ if not api_base:
+ raise ValueError(
+ "Milvus API base URL is required. Set MILVUS_API_BASE environment variable or pass api_base in litellm_params."
+ )
+
+ if api_base:
+ return api_base.rstrip("/")
+
+ return api_base
+
+ def transform_search_vector_store_request(
+ self,
+ vector_store_id: str,
+ query: Union[str, List[str]],
+ vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
+ api_base: str,
+ litellm_logging_obj: LiteLLMLoggingObj,
+ litellm_params: dict,
+ ) -> Tuple[str, Dict[str, Any]]:
+ """
+ Transform search request for Azure AI Search API
+
+ Generates embeddings using litellm.embeddings and constructs Azure AI Search request
+ """
+ # Convert query to string if it's a list
+ if isinstance(query, list):
+ query = " ".join(query)
+
+ # Get embedding model from litellm_params (required)
+ embedding_model = litellm_params.get("litellm_embedding_model")
+ if not embedding_model:
+ raise ValueError(
+ "embedding_model is required in litellm_params for Milvus. You can call any litellm embedding model."
+ "Example: litellm_params['embedding_model'] = 'azure/text-embedding-3-large'"
+ )
+
+ embedding_config = litellm_params.get("litellm_embedding_config", {})
+ if not embedding_config:
+ raise ValueError(
+ "embedding_config is required in litellm_params for Milvus. You can call any litellm embedding model."
+ "Example: litellm_params['embedding_config'] = {'api_base': 'https://krris-mh44uf7y-eastus2.cognitiveservices.azure.com/', 'api_key': 'os.environ/AZURE_API_KEY', 'api_version': '2025-09-01'}"
+ )
+
+ # Get top_k (number of results to return)
+ # Generate embedding for the query using litellm.embeddings
+ try:
+ embedding_response = litellm.embedding(
+ model=embedding_model,
+ input=[query],
+ **embedding_config,
+ )
+ query_vector = embedding_response.data[0]["embedding"]
+ except Exception as e:
+ raise Exception(f"Failed to generate embedding for query: {str(e)}")
+
+ # Azure AI Search endpoint for search
+ index_name = vector_store_id # vector_store_id is the index name
+ url = f"{api_base}/v2/vectordb/entities/search"
+
+ # Build the request body for Azure AI Search with vector search
+ request_body = {
+ "collectionName": index_name,
+ "data": [query_vector],
+ "annsField": "book_intro_vector",
+ **vector_store_search_optional_params,
+ }
+
+ #########################################################
+ # Update logging object with details of the request
+ #########################################################
+ litellm_logging_obj.model_call_details["input"] = query
+ litellm_logging_obj.model_call_details["embedding_model"] = embedding_model
+
+ return url, request_body
+
+ def transform_search_vector_store_response(
+ self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj
+ ) -> VectorStoreSearchResponse:
+ """
+ Transform Azure AI Search API response to standard vector store search response
+
+ Handles the format from Azure AI Search which returns:
+ {
+ "value": [
+ {
+ "id": "...",
+ "content": "...",
+ "distance": 0.95,
+ }
+ ]
+ }
+ """
+ try:
+ response_json = response.json()
+
+ # Extract results from Azure AI Search API response
+ results = response_json.get("data", [])
+
+ # Try to get text_field from optional_params first, then litellm_params
+ optional_params = litellm_logging_obj.model_call_details.get(
+ "optional_params", {}
+ )
+ text_field = optional_params.get("milvus_text_field", "")
+
+ # Fallback to litellm_params if not in optional_params
+
+ if not text_field:
+ text_field = litellm_logging_obj.model_call_details.get(
+ "litellm_params", {}
+ ).get("milvus_text_field", "")
+
+ # Transform results to standard format
+ search_results: List[VectorStoreSearchResult] = []
+ for result in results:
+ # Extract text content
+ text_content = result.get(text_field, "")
+
+ content = [
+ VectorStoreResultContent(
+ text=text_content,
+ type="text",
+ )
+ ]
+
+ # Get the search score (distance from the query vector)
+ score = result.get("distance", 0.0)
+
+ # Build attributes with all available metadata
+ # Exclude system fields and already-processed fields
+ attributes = {}
+ for key, value in result.items():
+ if key not in ["id", "content", "distance", text_field]:
+ attributes[key] = value
+
+ result_obj = VectorStoreSearchResult(
+ score=score,
+ content=content,
+ file_id=None,
+ filename=None,
+ attributes=attributes,
+ )
+ search_results.append(result_obj)
+
+ return VectorStoreSearchResponse(
+ object="vector_store.search_results.page",
+ search_query=litellm_logging_obj.model_call_details.get("input", ""),
+ data=search_results,
+ )
+
+ except Exception as e:
+ raise self.get_error_class(
+ error_message=str(e),
+ status_code=response.status_code,
+ headers=response.headers,
+ )
+
+ def transform_create_vector_store_request(
+ self,
+ vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams,
+ api_base: str,
+ ) -> Tuple[str, Dict]:
+ raise NotImplementedError
+
+ def transform_create_vector_store_response(
+ self, response: httpx.Response
+ ) -> VectorStoreCreateResponse:
+ raise NotImplementedError
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e1b0e602e92db578d32513c959fd1bcfce96eb01
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/__init__.py
@@ -0,0 +1,13 @@
+"""
+MiniMax LLM Provider
+"""
+
+from .text_to_speech.transformation import (
+ MinimaxException,
+ MinimaxTextToSpeechConfig,
+)
+
+__all__ = [
+ "MinimaxTextToSpeechConfig",
+ "MinimaxException",
+]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/chat/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/chat/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..eeeba74326d5dfab6897c1bb8507b2836d98de59
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/chat/__init__.py
@@ -0,0 +1,3 @@
+"""
+MiniMax OpenAI-compatible chat API
+"""
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..4095e57a8aeee2a7fa43cac4171dac09224e962b
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/chat/transformation.py
@@ -0,0 +1,101 @@
+"""
+MiniMax OpenAI transformation config - extends OpenAI chat config for MiniMax's OpenAI-compatible API
+"""
+from typing import List, Optional, Tuple
+
+import litellm
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
+
+
+class MinimaxChatConfig(OpenAIGPTConfig):
+ """
+ MiniMax OpenAI configuration that extends OpenAIGPTConfig.
+ MiniMax provides an OpenAI-compatible API at:
+ - International: https://api.minimax.io/v1
+ - China: https://api.minimaxi.com/v1
+
+ Supported models:
+ - MiniMax-M2.1
+ - MiniMax-M2.1-lightning
+ - MiniMax-M2
+ """
+
+ @staticmethod
+ def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
+ """
+ Get MiniMax API key from environment or parameters.
+ """
+ return api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key
+
+ @staticmethod
+ def get_api_base(
+ api_base: Optional[str] = None,
+ ) -> str:
+ """
+ Get MiniMax API base URL.
+ Defaults to international endpoint: https://api.minimax.io/v1
+ For China, set to: https://api.minimaxi.com/v1
+ """
+ return (
+ api_base
+ or get_secret_str("MINIMAX_API_BASE")
+ or "https://api.minimax.io/v1"
+ )
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ api_key: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: dict,
+ stream: Optional[bool] = None,
+ ) -> str:
+ """
+ Get the complete URL for MiniMax OpenAI API.
+ Override to ensure we use MiniMax's endpoint.
+ """
+ # Get the base URL (either provided or default MiniMax endpoint)
+ base_url = self.get_api_base(api_base=api_base)
+
+ # Ensure it ends with /chat/completions
+ if base_url.endswith("/chat/completions"):
+ return base_url
+ elif base_url.endswith("/v1"):
+ return f"{base_url}/chat/completions"
+ elif base_url.endswith("/"):
+ return f"{base_url}v1/chat/completions"
+ else:
+ return f"{base_url}/v1/chat/completions"
+
+ def remove_cache_control_flag_from_messages_and_tools(
+ self,
+ model: str,
+ messages: List[AllMessageValues],
+ tools: Optional[List[ChatCompletionToolParam]] = None,
+ ) -> Tuple[List[AllMessageValues], Optional[List[ChatCompletionToolParam]]]:
+ """
+ Override to preserve cache_control for MiniMax.
+ MiniMax supports cache_control - don't strip it.
+ """
+ # MiniMax supports cache_control, so return messages and tools unchanged
+ return messages, tools
+
+ def get_supported_openai_params(self, model: str) -> list:
+ """
+ Get supported OpenAI parameters for MiniMax.
+ Adds reasoning_split and thinking to the list of supported params.
+ """
+ base_params = super().get_supported_openai_params(model=model)
+ additional_params = ["reasoning_split"]
+
+ # Add thinking parameter if model supports reasoning
+ try:
+ if litellm.supports_reasoning(model=model, custom_llm_provider="minimax"):
+ additional_params.append("thinking")
+ except Exception:
+ pass
+
+ return base_params + additional_params
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/messages/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/messages/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..13ed6ad3863469417cdf92d927ec1f7ad28d2b1d
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/messages/transformation.py
@@ -0,0 +1,76 @@
+"""
+MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's Anthropic-compatible API
+"""
+from typing import Optional
+
+import litellm
+from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+ AnthropicMessagesConfig,
+)
+from litellm.secret_managers.main import get_secret_str
+
+
+class MinimaxMessagesConfig(AnthropicMessagesConfig):
+ """
+ MiniMax Anthropic configuration that extends AnthropicConfig.
+ MiniMax provides an Anthropic-compatible API at:
+ - International: https://api.minimax.io/anthropic
+ - China: https://api.minimaxi.com/anthropic
+
+ Supported models:
+ - MiniMax-M2.1
+ - MiniMax-M2.1-lightning
+ - MiniMax-M2
+ """
+
+ @property
+ def custom_llm_provider(self) -> Optional[str]:
+ return "minimax"
+
+ @staticmethod
+ def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
+ """
+ Get MiniMax API key from environment or parameters.
+ """
+ return api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key
+
+ @staticmethod
+ def get_api_base(
+ api_base: Optional[str] = None,
+ ) -> str:
+ """
+ Get MiniMax API base URL.
+ Defaults to international endpoint: https://api.minimax.io/anthropic
+ For China, set to: https://api.minimaxi.com/anthropic
+ """
+ return (
+ api_base
+ or get_secret_str("MINIMAX_API_BASE")
+ or "https://api.minimax.io/anthropic/v1/messages"
+ )
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ api_key: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: dict,
+ stream: Optional[bool] = None,
+ ) -> str:
+ """
+ Get the complete URL for MiniMax API.
+ Override to ensure we use MiniMax's endpoint, not Anthropic's.
+ """
+ # Get the base URL (either provided or default MiniMax endpoint)
+ base_url = self.get_api_base(api_base=api_base)
+
+ # If the base URL already includes the full path, return it
+ if base_url.endswith("/v1/messages"):
+ return base_url
+
+ # Otherwise append the messages endpoint
+ if base_url.endswith("/"):
+ return f"{base_url}v1/messages"
+ else:
+ return f"{base_url}/v1/messages"
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/text_to_speech/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/text_to_speech/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..bf4ac9010a49fedeb4c5aa489ad5ea81651a5714
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/text_to_speech/__init__.py
@@ -0,0 +1,7 @@
+"""
+MiniMax Text-to-Speech module
+"""
+
+from .transformation import MinimaxException, MinimaxTextToSpeechConfig
+
+__all__ = ["MinimaxTextToSpeechConfig", "MinimaxException"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/text_to_speech/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/text_to_speech/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a7d6897edc2abf48e3c5c701a9ef47ef716d359
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/text_to_speech/transformation.py
@@ -0,0 +1,414 @@
+"""
+MiniMax Text-to-Speech transformation
+
+Maps OpenAI TTS spec to MiniMax TTS API (WebSocket-based HTTP API)
+Reference: https://platform.minimax.io/docs
+"""
+
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
+
+import httpx
+from httpx import Headers
+
+import litellm
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.text_to_speech.transformation import (
+ BaseTextToSpeechConfig,
+ TextToSpeechRequestData,
+)
+from litellm.secret_managers.main import get_secret_str
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+ from litellm.types.llms.openai import HttpxBinaryResponseContent
+else:
+ LiteLLMLoggingObj = Any
+ HttpxBinaryResponseContent = Any
+
+
+class MinimaxException(BaseLLMException):
+ """Custom exception for MiniMax API errors"""
+
+ def __init__(
+ self,
+ status_code: int,
+ message: str,
+ headers: Optional[Union[dict, Headers]] = None,
+ ):
+ super().__init__(status_code=status_code, message=message, headers=headers)
+
+
+class MinimaxTextToSpeechConfig(BaseTextToSpeechConfig):
+ """
+ Configuration for MiniMax Text-to-Speech
+
+ Reference: https://platform.minimax.io/docs
+
+ MiniMax TTS API supports both WebSocket and HTTP endpoints.
+ This implementation uses the HTTP endpoint for simplicity.
+ """
+
+ TTS_BASE_URL = "https://api.minimax.io"
+ TTS_ENDPOINT_PATH = "/v1/t2a_v2"
+
+ # Voice mappings from OpenAI-style voices to MiniMax voice IDs
+ # MiniMax supports many voices, these are common mappings
+ VOICE_MAPPINGS = {
+ "alloy": "male-qn-qingse",
+ "echo": "male-qn-jingying",
+ "fable": "female-shaonv",
+ "onyx": "male-qn-badao",
+ "nova": "female-yujie",
+ "shimmer": "female-tianmei",
+ }
+
+ # Response format mappings from OpenAI to MiniMax
+ FORMAT_MAPPINGS = {
+ "mp3": "mp3",
+ "pcm": "pcm",
+ "wav": "wav",
+ "flac": "flac",
+ }
+
+ def get_supported_openai_params(self, model: str) -> list:
+ """
+ MiniMax TTS supports these OpenAI parameters
+ """
+ return ["voice", "response_format", "speed"]
+
+ def _extract_voice_id(self, voice: str) -> str:
+ """
+ Normalize the provided voice information into a MiniMax voice_id.
+ """
+ normalized_voice = voice.strip()
+ mapped_voice = self.VOICE_MAPPINGS.get(normalized_voice.lower())
+ return mapped_voice or normalized_voice
+
+ def _resolve_voice_id(
+ self,
+ voice: Optional[Union[str, Dict[str, Any]]],
+ params: Dict[str, Any],
+ ) -> str:
+ """
+ Determine the MiniMax voice_id based on provided voice input or parameters.
+ """
+ mapped_voice: Optional[str] = None
+
+ if isinstance(voice, str) and voice.strip():
+ mapped_voice = self._extract_voice_id(voice)
+ elif isinstance(voice, dict):
+ for key in ("voice_id", "id", "name"):
+ candidate = voice.get(key)
+ if isinstance(candidate, str) and candidate.strip():
+ mapped_voice = self._extract_voice_id(candidate)
+ break
+ elif voice is not None:
+ mapped_voice = self._extract_voice_id(str(voice))
+
+ if mapped_voice is None:
+ voice_override = params.pop("voice_id", None)
+ if isinstance(voice_override, str) and voice_override.strip():
+ mapped_voice = self._extract_voice_id(voice_override)
+
+ if mapped_voice is None:
+ # Default to a common voice if not specified
+ mapped_voice = "male-qn-qingse"
+
+ return mapped_voice
+
+ def map_openai_params(
+ self,
+ model: str,
+ optional_params: Dict,
+ voice: Optional[Union[str, Dict]] = None,
+ drop_params: bool = False,
+ kwargs: Optional[Dict[str, Any]] = None,
+ ) -> Tuple[Optional[str], Dict]:
+ """
+ Map OpenAI parameters to MiniMax TTS parameters
+ """
+ mapped_params: Dict[str, Any] = {}
+
+ # Work on a copy so we don't mutate the caller's dictionary
+ params = dict(optional_params) if optional_params else {}
+
+ # Extract voice identifier
+ mapped_voice = self._resolve_voice_id(voice, params)
+
+ # Response/output format
+ response_format = params.pop("response_format", None)
+ if isinstance(response_format, str):
+ mapped_format = self.FORMAT_MAPPINGS.get(response_format, "mp3")
+ mapped_params["format"] = mapped_format
+ else:
+ mapped_params["format"] = "mp3" # Default format
+
+ # Speed parameter (MiniMax supports speed from 0.5 to 2.0)
+ speed = params.pop("speed", None)
+ if speed is not None:
+ try:
+ speed_value = float(speed)
+ # Clamp speed to MiniMax's supported range
+ speed_value = max(0.5, min(2.0, speed_value))
+ mapped_params["speed"] = speed_value
+ except (TypeError, ValueError):
+ mapped_params["speed"] = 1.0
+ else:
+ mapped_params["speed"] = 1.0
+
+ # Instructions parameter is OpenAI-specific; omit to prevent API errors
+ params.pop("instructions", None)
+
+ # Store voice_id for later use in request construction
+ mapped_params["voice_id"] = mapped_voice
+
+ # Handle extra_body for additional MiniMax-specific parameters
+ extra_body = params.pop("extra_body", None)
+ if isinstance(extra_body, dict):
+ for key, value in extra_body.items():
+ if value is not None:
+ mapped_params[key] = value
+
+ # Pass through any remaining parameters
+ for key, value in params.items():
+ if value is not None:
+ mapped_params[key] = value
+
+ return mapped_voice, mapped_params
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ """
+ Validate MiniMax environment and set up authentication headers
+ """
+ api_key = api_key or litellm.api_key or get_secret_str("MINIMAX_API_KEY")
+
+ if api_key is None:
+ raise ValueError(
+ "MiniMax API key is required. Set MINIMAX_API_KEY environment variable or pass api_key parameter."
+ )
+
+ headers.update(
+ {
+ "Authorization": f"Bearer {api_key}",
+ "Content-Type": "application/json",
+ }
+ )
+
+ return headers
+
+ def get_error_class(
+ self, error_message: str, status_code: int, headers: Union[dict, Headers]
+ ) -> BaseLLMException:
+ return MinimaxException(
+ message=error_message, status_code=status_code, headers=headers
+ )
+
+ def transform_text_to_speech_request(
+ self,
+ model: str,
+ input: str,
+ voice: Optional[str],
+ optional_params: Dict,
+ litellm_params: Dict,
+ headers: dict,
+ ) -> TextToSpeechRequestData:
+ """
+ Build the MiniMax TTS request payload.
+
+ MiniMax uses a different structure than OpenAI:
+ - model: The TTS model to use
+ - text: The input text
+ - voice_setting: Voice configuration
+ - audio_setting: Audio output configuration
+ """
+ params = dict(optional_params) if optional_params else {}
+
+ # Extract parameters
+ voice_id = params.pop("voice_id", voice or "male-qn-qingse")
+ speed = params.pop("speed", 1.0)
+ audio_format = params.pop("format", "mp3")
+
+ # Extract additional voice settings
+ vol = params.pop("vol", 1.0) # Volume (0.1 to 10)
+ pitch = params.pop("pitch", 0) # Pitch adjustment (-12 to 12)
+
+ # Extract audio settings
+ sample_rate = params.pop("sample_rate", 32000) # 16000, 24000, 32000
+ bitrate = params.pop(
+ "bitrate", 128000
+ ) # For MP3: 64000, 128000, 192000, 256000
+ channel = params.pop("channel", 1) # 1 for mono, 2 for stereo
+
+ # Output format: 'url' or 'hex' (default is 'hex')
+ output_format = params.pop("output_format", "hex")
+
+ request_body: Dict[str, Any] = {
+ "model": model,
+ "text": input,
+ "stream": False, # HTTP endpoint doesn't support streaming
+ "output_format": output_format, # 'url' or 'hex'
+ "voice_setting": {
+ "voice_id": voice_id,
+ "speed": speed,
+ "vol": vol,
+ "pitch": pitch,
+ },
+ "audio_setting": {
+ "sample_rate": sample_rate,
+ "bitrate": bitrate,
+ "format": audio_format,
+ "channel": channel,
+ },
+ }
+
+ # Handle any remaining parameters from extra_body
+ extra_body = params.pop("extra_body", None)
+ if isinstance(extra_body, dict):
+ for key, value in extra_body.items():
+ if value is not None and key not in request_body:
+ request_body[key] = value
+
+ return TextToSpeechRequestData(
+ dict_body=request_body,
+ headers={"Content-Type": "application/json"},
+ )
+
+ def transform_text_to_speech_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> "HttpxBinaryResponseContent":
+ """
+ Transform MiniMax response to standard format.
+
+ MiniMax returns JSON with base64-encoded audio data:
+ {
+ "base_resp": {"status_code": 0, "status_msg": "success"},
+ "audio_file": "",
+ "extra_info": {...}
+ }
+
+ We need to decode the base64 audio and return it as binary content.
+ """
+ import base64
+ import json
+
+ from litellm.types.llms.openai import HttpxBinaryResponseContent
+
+ try:
+ # Parse JSON response
+ response_json = raw_response.json()
+
+ # MiniMax API response format check
+ # The API can return different structures:
+ # 1. {"data": {"audio": "..."}, "status": 0, ...} for HTTP endpoint
+ # 2. {"base_resp": {"status_code": 0, ...}, "audio_file": "..."} for older versions
+
+ # Check for errors - MiniMax uses "status" field in HTTP endpoint response
+ # status: 0 = success, 2 = invalid api key, etc.
+ status = response_json.get("status")
+ if status is not None and status != 0:
+ ced = response_json.get("ced", "Unknown error")
+ error_detail = ced if ced else f"API returned status {status}"
+ raise MinimaxException(
+ status_code=raw_response.status_code,
+ message=f"MiniMax TTS error: {error_detail}",
+ headers=dict(raw_response.headers),
+ )
+
+ # Extract audio data
+ # MiniMax returns audio in "data" field
+ data = response_json.get("data", {})
+
+ # Check if response contains a URL (output_format='url')
+ audio_url = data.get("audio_url", None)
+ if audio_url:
+ # If URL format is used, we need to fetch the audio from the URL
+ # For now, return a response indicating URL mode (TODO: fetch audio from URL)
+ raise MinimaxException(
+ status_code=500,
+ message=f"URL output format is not yet supported. Use 'hex' format or fetch from URL: {audio_url}",
+ headers=dict(raw_response.headers),
+ )
+
+ # Get hex-encoded audio data
+ audio_hex = data.get("audio", "") or response_json.get("audio_file", "")
+
+ if not audio_hex:
+ raise MinimaxException(
+ status_code=500,
+ message=f"No audio data in MiniMax response. Response keys: {list(response_json.keys())}",
+ headers=dict(raw_response.headers),
+ )
+
+ # MiniMax returns hex-encoded audio by default
+ # Try hex decoding first, fall back to base64 if that fails
+ try:
+ audio_bytes = bytes.fromhex(audio_hex)
+ except ValueError:
+ # If hex decoding fails, try base64 (for older API versions)
+ try:
+ audio_bytes = base64.b64decode(audio_hex)
+ except Exception as e:
+ raise MinimaxException(
+ status_code=500,
+ message=f"Failed to decode audio data: {str(e)}",
+ headers=dict(raw_response.headers),
+ )
+
+ # Create a new response with binary audio content
+ # We need to create a response that contains the decoded audio bytes
+ # Remove gzip encoding headers to avoid decompression issues
+ clean_headers = dict(raw_response.headers)
+ clean_headers.pop("content-encoding", None)
+ clean_headers.pop("transfer-encoding", None)
+ clean_headers["content-length"] = str(len(audio_bytes))
+
+ # Create a new response object with the binary content
+ binary_response = httpx.Response(
+ status_code=200,
+ headers=clean_headers,
+ content=audio_bytes,
+ request=raw_response.request,
+ )
+
+ return HttpxBinaryResponseContent(binary_response)
+
+ except json.JSONDecodeError as e:
+ raise MinimaxException(
+ status_code=500,
+ message=f"Failed to parse MiniMax response: {str(e)}",
+ headers=dict(raw_response.headers),
+ )
+ except Exception as e:
+ if isinstance(e, MinimaxException):
+ raise
+ raise MinimaxException(
+ status_code=500,
+ message=f"Error processing MiniMax response: {str(e)}",
+ headers=dict(raw_response.headers),
+ )
+
+ def get_complete_url(
+ self,
+ model: str,
+ api_base: Optional[str],
+ litellm_params: dict,
+ ) -> str:
+ """
+ Construct the MiniMax endpoint URL.
+ """
+ base_url = api_base or get_secret_str("MINIMAX_API_BASE") or self.TTS_BASE_URL
+ base_url = base_url.rstrip("/")
+
+ # MiniMax uses a simple endpoint path
+ url = f"{base_url}{self.TTS_ENDPOINT_PATH}"
+
+ return url
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/audio_transcription/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/audio_transcription/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..8c6d604acb4b5384ad80f421df1b41d522ab3b6c
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/audio_transcription/transformation.py
@@ -0,0 +1,159 @@
+"""
+Support for Mistral Voxtral audio transcription via ``/v1/audio/transcriptions``.
+
+API reference: https://docs.mistral.ai/api/#tag/audio/operation/audio_transcriptions_v1_audio_transcriptions_post
+"""
+
+from typing import List, Optional, Union
+
+import httpx
+
+from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
+from litellm.llms.base_llm.audio_transcription.transformation import (
+ AudioTranscriptionRequestData,
+ BaseAudioTranscriptionConfig,
+)
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import (
+ AllMessageValues,
+ OpenAIAudioTranscriptionOptionalParams,
+)
+from litellm.types.utils import FileTypes, TranscriptionResponse
+
+
+class MistralAudioTranscriptionException(BaseLLMException):
+ pass
+
+
+class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
+ def get_supported_openai_params(
+ self, model: str
+ ) -> List[OpenAIAudioTranscriptionOptionalParams]:
+ return [
+ "language",
+ "temperature",
+ "timestamp_granularities",
+ "response_format",
+ ]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ supported_params = self.get_supported_openai_params(model)
+ for k, v in non_default_params.items():
+ if k in supported_params:
+ optional_params[k] = v
+ return optional_params
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ api_key: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: dict,
+ stream: Optional[bool] = None,
+ ) -> str:
+ api_base = (
+ "https://api.mistral.ai/v1" if api_base is None else api_base.rstrip("/")
+ )
+ return f"{api_base}/audio/transcriptions"
+
+ def get_error_class(
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+ ) -> BaseLLMException:
+ return MistralAudioTranscriptionException(
+ message=error_message,
+ status_code=status_code,
+ headers=headers,
+ )
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ if api_key is None:
+ api_key = get_secret_str("MISTRAL_API_KEY")
+
+ default_headers = {
+ "Authorization": f"Bearer {api_key}",
+ "accept": "application/json",
+ }
+ default_headers.update(headers or {})
+ return default_headers
+
+ def transform_audio_transcription_request(
+ self,
+ model: str,
+ audio_file: FileTypes,
+ optional_params: dict,
+ litellm_params: dict,
+ ) -> AudioTranscriptionRequestData:
+ processed_audio = process_audio_file(audio_file)
+
+ form_fields: dict = {
+ "model": model,
+ }
+
+ # OpenAI-compatible params
+ for key in self.get_supported_openai_params(model):
+ value = optional_params.get(key)
+ if value is not None:
+ form_fields[key] = value
+
+ # Mistral-specific params (e.g. diarize)
+ provider_specific_params = self.get_provider_specific_params(
+ model=model,
+ optional_params=optional_params,
+ openai_params=self.get_supported_openai_params(model),
+ )
+ for key, value in provider_specific_params.items():
+ form_fields[key] = (
+ str(value).lower() if isinstance(value, bool) else str(value)
+ )
+
+ files = {
+ "file": (
+ processed_audio.filename,
+ processed_audio.file_content,
+ processed_audio.content_type,
+ )
+ }
+
+ return AudioTranscriptionRequestData(data=form_fields, files=files)
+
+ def transform_audio_transcription_response(
+ self,
+ raw_response: httpx.Response,
+ ) -> TranscriptionResponse:
+ try:
+ response_json = raw_response.json()
+ except Exception:
+ raise MistralAudioTranscriptionException(
+ message=raw_response.text,
+ status_code=raw_response.status_code,
+ headers=raw_response.headers,
+ )
+
+ text = response_json.get("text") or ""
+ response = TranscriptionResponse(text=text)
+
+ # Preserve Mistral-specific fields (e.g. diarization segments)
+ if "segments" in response_json:
+ response["segments"] = response_json["segments"]
+ if "language" in response_json:
+ response["language"] = response_json["language"]
+
+ response._hidden_params = response_json
+ return response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..23fbe467fc8598e3c1422c8e36fdffe7883f57db
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/chat/transformation.py
@@ -0,0 +1,686 @@
+"""
+Transformation logic from OpenAI /v1/chat/completion format to Mistral's /chat/completion format.
+
+Why separate file? Make it easy to see how transformation works
+
+Docs - https://docs.mistral.ai/api/
+"""
+
+from typing import (
+ Any,
+ AsyncIterator,
+ Coroutine,
+ Iterator,
+ List,
+ Literal,
+ Optional,
+ Tuple,
+ Union,
+ cast,
+ get_type_hints,
+ overload,
+)
+
+import httpx
+
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+ handle_messages_with_content_list_to_str_conversion,
+ strip_none_values_from_message,
+)
+from litellm.llms.openai.chat.gpt_transformation import (
+ OpenAIGPTConfig,
+ OpenAIChatCompletionStreamingHandler,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.mistral import MistralThinkingBlock, MistralToolCallMessage
+from litellm.types.llms.openai import AllMessageValues
+from litellm.types.utils import ModelResponse, ModelResponseStream
+from litellm.utils import convert_to_model_response_object
+
+
+class MistralConfig(OpenAIGPTConfig):
+ """
+ Reference: https://docs.mistral.ai/api/
+
+ The class `MistralConfig` provides configuration for the Mistral's Chat API interface. Below are the parameters:
+
+ - `temperature` (number or null): Defines the sampling temperature to use, varying between 0 and 2. API Default - 0.7.
+
+ - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling. API Default - 1.
+
+ - `max_tokens` (integer or null): This optional parameter helps to set the maximum number of tokens to generate in the chat completion. API Default - null.
+
+ - `tools` (list or null): A list of available tools for the model. Use this to specify functions for which the model can generate JSON inputs.
+
+ - `tool_choice` (string - 'auto'/'any'/'none' or null): Specifies if/how functions are called. If set to none the model won't call a function and will generate a message instead. If set to auto the model can choose to either generate a message or call a function. If set to any the model is forced to call a function. Default - 'auto'.
+
+ - `stop` (string or array of strings): Stop generation if this token is detected. Or if one of these tokens is detected when providing an array
+
+ - `random_seed` (integer or null): The seed to use for random sampling. If set, different calls will generate deterministic results.
+
+ - `safe_prompt` (boolean): Whether to inject a safety prompt before all conversations. API Default - 'false'.
+
+ - `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
+ """
+
+ temperature: Optional[int] = None
+ top_p: Optional[int] = None
+ max_tokens: Optional[int] = None
+ tools: Optional[list] = None
+ tool_choice: Optional[Literal["auto", "any", "none"]] = None
+ random_seed: Optional[int] = None
+ safe_prompt: Optional[bool] = None
+ response_format: Optional[dict] = None
+ stop: Optional[Union[str, list]] = None
+
+ def __init__(
+ self,
+ temperature: Optional[int] = None,
+ top_p: Optional[int] = None,
+ max_tokens: Optional[int] = None,
+ tools: Optional[list] = None,
+ tool_choice: Optional[Literal["auto", "any", "none"]] = None,
+ random_seed: Optional[int] = None,
+ safe_prompt: Optional[bool] = None,
+ response_format: Optional[dict] = None,
+ stop: Optional[Union[str, list]] = None,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return super().get_config()
+
+ def get_supported_openai_params(self, model: str) -> List[str]:
+ supported_params = [
+ "stream",
+ "temperature",
+ "top_p",
+ "max_tokens",
+ "max_completion_tokens",
+ "tools",
+ "tool_choice",
+ "seed",
+ "stop",
+ "response_format",
+ "parallel_tool_calls",
+ ]
+
+ # Add reasoning support for magistral models
+ if "magistral" in model.lower():
+ supported_params.extend(["thinking", "reasoning_effort"])
+
+ return supported_params
+
+ def _map_tool_choice(self, tool_choice: str) -> str:
+ if tool_choice == "auto" or tool_choice == "none":
+ return tool_choice
+ elif tool_choice == "required":
+ return "any"
+ else: # openai 'tool_choice' object param not supported by Mistral API
+ return "any"
+
+ @staticmethod
+ def _get_mistral_reasoning_system_prompt() -> str:
+ """
+ Returns the system prompt for Mistral reasoning models.
+ Based on Mistral's documentation: https://huggingface.co/mistralai/Magistral-Small-2506
+
+ Mistral recommends the following system prompt for reasoning:
+ """
+ return """
+ [SYSTEM_PROMPT]system_prompt
+ A user will ask you to solve a task. You should first draft your thinking process (inner monologue) until you have derived the final answer. Afterwards, write a self-contained summary of your thoughts (i.e. your summary should be succinct but contain all the critical steps you needed to reach the conclusion). You should use Markdown to format your response. Write both your thoughts and summary in the same language as the task posed by the user. NEVER use \boxed{} in your response.
+
+ Your thinking process must follow the template below:
+
+ Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct answer.
+
+
+ Here, provide a concise summary that reflects your reasoning and presents a clear final answer to the user. Don't mention that this is a summary.
+
+ Problem:
+
+ [/SYSTEM_PROMPT][INST]user_message[/INST]
+ reasoning_traces
+
+ assistant_response[INST]user_message[/INST]
+ """
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ for param, value in non_default_params.items():
+ if param == "max_tokens":
+ optional_params["max_tokens"] = value
+ if (
+ param == "max_completion_tokens"
+ ): # max_completion_tokens should take priority
+ optional_params["max_tokens"] = value
+ if param == "tools":
+ # Clean tools to remove problematic schema fields for Mistral API
+ optional_params["tools"] = self._clean_tool_schema_for_mistral(value)
+ if param == "stream" and value is True:
+ optional_params["stream"] = value
+ if param == "temperature":
+ optional_params["temperature"] = value
+ if param == "top_p":
+ optional_params["top_p"] = value
+ if param == "stop":
+ optional_params["stop"] = value
+ if param == "tool_choice" and isinstance(value, str):
+ optional_params["tool_choice"] = self._map_tool_choice(
+ tool_choice=value
+ )
+ if param == "seed":
+ optional_params["extra_body"] = {"random_seed": value}
+ if param == "response_format":
+ optional_params["response_format"] = value
+ if param == "reasoning_effort" and "magistral" in model.lower():
+ # Flag that we need to add reasoning system prompt
+ optional_params["_add_reasoning_prompt"] = True
+ if param == "thinking" and "magistral" in model.lower():
+ # Flag that we need to add reasoning system prompt
+ optional_params["_add_reasoning_prompt"] = True
+ if param == "parallel_tool_calls":
+ optional_params["parallel_tool_calls"] = value
+ return optional_params
+
+ def _get_openai_compatible_provider_info(
+ self, api_base: Optional[str], api_key: Optional[str]
+ ) -> Tuple[str, Optional[str]]:
+ # mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai
+ api_base = (
+ api_base
+ or get_secret_str("MISTRAL_AZURE_API_BASE") # for Azure AI Mistral
+ or "https://api.mistral.ai/v1"
+ ) # type: ignore
+
+ # if api_base does not end with /v1 we add it
+ if api_base is not None and not api_base.endswith(
+ "/v1"
+ ): # Mistral always needs a /v1 at the end
+ api_base = api_base + "/v1"
+ dynamic_api_key = (
+ api_key
+ or get_secret_str("MISTRAL_AZURE_API_KEY") # for Azure AI Mistral
+ or get_secret_str("MISTRAL_API_KEY")
+ )
+ return api_base, dynamic_api_key
+
+ # fmt: off
+
+ @overload
+ def _transform_messages(
+ self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
+ ) -> Coroutine[Any, Any, List[AllMessageValues]]:
+ ...
+
+ @overload
+ def _transform_messages(
+ self,
+ messages: List[AllMessageValues],
+ model: str,
+ is_async: Literal[False] = False,
+ ) -> List[AllMessageValues]:
+ ...
+ # fmt: on
+
+ def _transform_messages(
+ self, messages: List[AllMessageValues], model: str, is_async: bool = False
+ ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
+ """
+ - handles scenario where content is list and not string
+ - content list is just text, and no images
+ - if image passed in, then just return as is (user-intended)
+ - if `name` is passed, then drop it for mistral API: https://github.com/BerriAI/litellm/issues/6696
+
+ Motivation: mistral api doesn't support content as a list.
+ The above statement is not valid now. Need to plan to remove all the #1,2,3
+ Mistral API supports content as a list.
+ """
+ ## 1. If 'image_url' or 'file' in content, then transform with base class and mistral-specific handling
+ for m in messages:
+ _content_block = m.get("content")
+ if _content_block and isinstance(_content_block, list):
+ if any(c.get("type") in ["image_url", "file"] for c in _content_block):
+ if is_async:
+ return self._transform_messages_async(messages, model)
+ else:
+ messages = self._transform_messages_sync(messages, model)
+ return messages
+
+ ## 2. If content is list, then convert to string
+ messages = handle_messages_with_content_list_to_str_conversion(messages)
+
+ ## 3. Handle name in message
+ new_messages: List[AllMessageValues] = []
+ for m in messages:
+ m = MistralConfig._handle_name_in_message(m)
+ m = MistralConfig._handle_tool_call_message(m)
+ if MistralConfig._is_empty_assistant_message(m):
+ continue
+ m = strip_none_values_from_message(m) # prevents 'extra_forbidden' error
+ new_messages.append(m)
+
+ if is_async:
+ return super()._transform_messages(new_messages, model, True)
+ else:
+ return super()._transform_messages(new_messages, model, False)
+
+ async def _transform_messages_async(
+ self, messages: List[AllMessageValues], model: str
+ ) -> List[AllMessageValues]:
+ """
+ Handle modification of messages for Mistral API in an async context.
+ """
+ # Call parent async method to handle basic transformations
+ # and then apply Mistral-specific handling for files
+ messages = await super()._transform_messages(messages, model, True)
+ messages = self._handle_message_with_file(messages)
+ return messages
+
+ def _transform_messages_sync(
+ self, messages: List[AllMessageValues], model: str
+ ) -> List[AllMessageValues]:
+ """Handle modification of messages for Mistral API in a sync context."""
+ # Call parent sync method to handle basic transformations
+ # and then apply Mistral-specific handling for files
+ # This is the sync version of the async method above
+ messages = super()._transform_messages(messages, model, False)
+ messages = self._handle_message_with_file(messages)
+ return messages
+
+ def _handle_message_with_file(
+ self, messages: List[AllMessageValues]
+ ) -> List[AllMessageValues]:
+ """
+ Mistral API supports only 'file_id' in message content with type 'file'.
+ """
+ for m in messages:
+ _content_block = m.get("content")
+ if _content_block and isinstance(_content_block, list):
+ if any(c.get("type") == "file" for c in _content_block):
+ # If file content is present, we get file_id from 'file' attribute of content block
+ # then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it.
+ file_contents = [
+ c for c in _content_block if c.get("type") == "file"
+ ]
+ for file_content in file_contents:
+ file_id = file_content.get("file", {}).get("file_id")
+ if file_id:
+ # Replace 'file' with 'file_id'
+ file_content["file_id"] = file_id # type: ignore
+ file_content.pop("file", None)
+ return messages
+
+ def _add_reasoning_system_prompt_if_needed(
+ self, messages: List[AllMessageValues], optional_params: dict
+ ) -> List[AllMessageValues]:
+ """
+ Add reasoning system prompt for Mistral magistral models when reasoning_effort is specified.
+ """
+ if not optional_params.get("_add_reasoning_prompt", False):
+ return messages
+
+ # Check if there's already a system message
+ has_system_message = any(msg.get("role") == "system" for msg in messages)
+
+ if has_system_message:
+ # Prepend reasoning instructions to existing system message
+ for i, msg in enumerate(messages):
+ if msg.get("role") == "system":
+ existing_content = msg.get("content", "")
+ reasoning_prompt = self._get_mistral_reasoning_system_prompt()
+
+ # Handle both string and list content, preserving original format
+ if isinstance(existing_content, str):
+ # String content - prepend reasoning prompt
+ new_content: Union[
+ str, list
+ ] = f"{reasoning_prompt}\n\n{existing_content}"
+ elif isinstance(existing_content, list):
+ # List content - prepend reasoning prompt as text block
+ new_content = [
+ {"type": "text", "text": reasoning_prompt + "\n\n"}
+ ] + existing_content
+ else:
+ # Fallback for any other type - convert to string
+ new_content = f"{reasoning_prompt}\n\n{str(existing_content)}"
+
+ messages[i] = cast(
+ AllMessageValues, {**msg, "content": new_content}
+ )
+ break
+ else:
+ # Add new system message with reasoning instructions
+ reasoning_message: AllMessageValues = cast(
+ AllMessageValues,
+ {
+ "role": "system",
+ "content": self._get_mistral_reasoning_system_prompt(),
+ },
+ )
+ messages = [reasoning_message] + messages
+
+ # Remove the internal flag
+ optional_params.pop("_add_reasoning_prompt", None)
+ return messages
+
+ @classmethod
+ def _clean_tool_schema_for_mistral(cls, tools: list) -> list:
+ """
+ Clean tool schemas to remove fields that cause issues with Mistral API.
+
+ Removes:
+ - $id and $schema fields (cause grammar validation errors)
+ - additionalProperties=False (causes OpenAI API schema errors)
+ - strict field (not supported by Mistral)
+
+ Args:
+ tools: List of tool definitions
+ max_depth: Maximum recursion depth for schema cleaning (default: 10)
+
+ Returns:
+ Cleaned tools list
+ """
+ if not tools:
+ return tools
+
+ import copy
+
+ from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
+ from litellm.utils import _remove_json_schema_refs
+
+ cleaned_tools = copy.deepcopy(tools)
+
+ # Apply all cleaning functions with max_depth protection
+ cleaned_tools = _remove_json_schema_refs(
+ cleaned_tools, max_depth=DEFAULT_MAX_RECURSE_DEPTH
+ )
+
+ return cleaned_tools
+
+ @classmethod
+ def _handle_name_in_message(cls, message: AllMessageValues) -> AllMessageValues:
+ """
+ Mistral API only supports `name` in tool messages
+
+ If role == tool, then we keep `name` if it's not an empty string
+ Otherwise, we drop `name`
+ """
+ _name = message.get("name") # type: ignore
+
+ if _name is not None:
+ # Remove name if not a tool message
+ if message["role"] != "tool":
+ message.pop("name", None) # type: ignore
+ # For tool messages, remove name if it's an empty string
+ elif isinstance(_name, str) and len(_name.strip()) == 0:
+ message.pop("name", None) # type: ignore
+
+ return message
+
+ @classmethod
+ def _handle_tool_call_message(cls, message: AllMessageValues) -> AllMessageValues:
+ """
+ Mistral API only supports tool_calls in Messages in `MistralToolCallMessage` spec
+ """
+ _tool_calls = message.get("tool_calls")
+ mistral_tool_calls: List[MistralToolCallMessage] = []
+ if _tool_calls is not None and isinstance(_tool_calls, list):
+ for _tool in _tool_calls:
+ _tool_call_message = MistralToolCallMessage(
+ id=_tool.get("id"),
+ type="function",
+ function=_tool.get("function"), # type: ignore
+ )
+ mistral_tool_calls.append(_tool_call_message)
+ message["tool_calls"] = mistral_tool_calls # type: ignore
+ return message
+
+ @classmethod
+ def _is_empty_assistant_message(cls, message: AllMessageValues) -> bool:
+ """
+ Mistral API does not support empty string in assistant content.
+ """
+ from litellm.types.llms.openai import ChatCompletionAssistantMessage
+
+ set_keys = get_type_hints(ChatCompletionAssistantMessage).keys()
+
+ all_expected_values_are_empty = True
+ for key in set_keys:
+ if key != "role" and message.get(key) is not None:
+ if key == "content" and message.get(key) == "":
+ continue
+ else:
+ all_expected_values_are_empty = False
+ break
+ return all_expected_values_are_empty
+
+ @staticmethod
+ def _handle_empty_content_response(response_data: dict) -> dict:
+ """
+ Handle Mistral-specific behavior where empty string content should be converted to None.
+
+ Mistral API sometimes returns empty string content ('') instead of null,
+ which can cause issues with downstream processing.
+
+ Args:
+ response_data: The raw response data from Mistral API
+
+ Returns:
+ dict: The response data with empty string content converted to None
+ """
+ if response_data.get("choices") and len(response_data["choices"]) > 0:
+ for choice in response_data["choices"]:
+ if choice.get("message") and choice["message"].get("content") == "":
+ choice["message"]["content"] = None
+ return response_data
+
+ @staticmethod
+ def _convert_thinking_block_to_reasoning_content(
+ thinking_blocks: MistralThinkingBlock,
+ ) -> str:
+ """
+ Convert Mistral thinking blocks to reasoning content.
+ """
+ return "\n".join(
+ [block.get("text", "") for block in thinking_blocks["thinking"]]
+ )
+
+ @staticmethod
+ def _handle_content_list_to_str_conversion(response_data: dict) -> dict:
+ """
+ Handle Mistral's content list format and extract thinking content.
+
+ Map mistral's content list to string and extract thinking blocks:
+ - Thinking block -> reasoning_content field
+ - Text block -> content field
+ """
+
+ if response_data.get("choices") and len(response_data["choices"]) > 0:
+ for choice in response_data["choices"]:
+ if choice.get("message") and choice["message"].get("content"):
+ content = choice["message"]["content"]
+
+ # Only process if content is a list
+ if isinstance(content, list):
+ thinking_content = ""
+ text_content = ""
+
+ # Process each content block
+ for block in content:
+ if block.get("type") == "thinking":
+ thinking_blocks = block.get("thinking", [])
+ thinking_texts = []
+ for thinking_block in thinking_blocks:
+ if thinking_block.get("type") == "text":
+ thinking_texts.append(
+ thinking_block.get("text", "")
+ )
+ thinking_content = "\n".join(thinking_texts)
+ elif block.get("type") == "text":
+ text_content = block.get("text", "")
+
+ # Set the extracted content
+ choice["message"]["content"] = text_content
+ if thinking_content:
+ choice["message"]["reasoning_content"] = thinking_content
+
+ return response_data
+
+ def transform_request(
+ self,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ headers: dict,
+ ) -> dict:
+ """
+ Transform the overall request to be sent to the API.
+ For magistral models, adds reasoning system prompt when reasoning_effort is specified.
+
+ Returns:
+ dict: The transformed request. Sent as the body of the API call.
+ """
+ # Add reasoning system prompt if needed (for magistral models)
+ if "magistral" in model.lower() and optional_params.get(
+ "_add_reasoning_prompt", False
+ ):
+ messages = self._add_reasoning_system_prompt_if_needed(
+ messages, optional_params
+ )
+
+ # Call parent transform_request which handles _transform_messages
+ return super().transform_request(
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ headers=headers,
+ )
+
+ def transform_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ model_response: ModelResponse,
+ logging_obj: LiteLLMLoggingObj,
+ request_data: dict,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: Any,
+ api_key: Optional[str] = None,
+ json_mode: Optional[bool] = None,
+ ) -> ModelResponse:
+ """
+ Transform the raw response from Mistral API.
+ Handles Mistral-specific behavior like converting empty string content to None
+ and extracting thinking content from content lists.
+ """
+ logging_obj.post_call(original_response=raw_response.text)
+ logging_obj.model_call_details["response_headers"] = raw_response.headers
+
+ # Handle Mistral-specific response transformations
+ response_data = raw_response.json()
+ response_data = self._handle_empty_content_response(response_data)
+ response_data = self._handle_content_list_to_str_conversion(response_data)
+
+ final_response_obj = cast(
+ ModelResponse,
+ convert_to_model_response_object(
+ response_object=response_data,
+ model_response_object=model_response,
+ hidden_params={"headers": raw_response.headers},
+ _response_headers=dict(raw_response.headers),
+ ),
+ )
+
+ return final_response_obj
+
+ def get_model_response_iterator(
+ self,
+ streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
+ sync_stream: bool,
+ json_mode: Optional[bool] = False,
+ ):
+ return MistralChatResponseIterator(
+ streaming_response=streaming_response,
+ sync_stream=sync_stream,
+ json_mode=json_mode,
+ )
+
+
+class MistralChatResponseIterator(OpenAIChatCompletionStreamingHandler):
+ def chunk_parser(self, chunk: dict) -> ModelResponseStream:
+ try:
+ for choice in chunk.get("choices", []):
+ delta = choice.get("delta", {})
+ content = delta.get("content")
+ if isinstance(content, list):
+ (
+ normalized_text,
+ thinking_blocks,
+ reasoning_content,
+ ) = self._normalize_content_blocks(content)
+ delta["content"] = normalized_text
+ if thinking_blocks:
+ delta["thinking_blocks"] = thinking_blocks
+ delta["reasoning_content"] = reasoning_content
+ else:
+ delta.pop("thinking_blocks", None)
+ delta.pop("reasoning_content", None)
+ except Exception:
+ # Fall back to default parsing if custom handling fails
+ return super().chunk_parser(chunk)
+
+ return super().chunk_parser(chunk)
+
+ @staticmethod
+ def _normalize_content_blocks(
+ content_blocks: List[dict],
+ ) -> Tuple[Optional[str], List[dict], Optional[str]]:
+ """
+ Convert Mistral magistral content blocks into OpenAI-compatible content + thinking_blocks.
+ """
+ text_segments: List[str] = []
+ thinking_blocks: List[dict] = []
+ reasoning_segments: List[str] = []
+
+ for block in content_blocks:
+ block_type = block.get("type")
+ if block_type == "thinking":
+ mistral_thinking = block.get("thinking", [])
+ thinking_text_parts: List[str] = []
+ for thinking_block in mistral_thinking:
+ if thinking_block.get("type") == "text":
+ thinking_text_parts.append(thinking_block.get("text", ""))
+ thinking_text = "".join(thinking_text_parts)
+ if thinking_text:
+ reasoning_segments.append(thinking_text)
+ thinking_blocks.append(
+ {
+ "type": "thinking",
+ "thinking": thinking_text,
+ "signature": "mistral",
+ }
+ )
+ elif block_type == "text":
+ text_segments.append(block.get("text", ""))
+
+ normalized_text = "".join(text_segments) if text_segments else None
+ reasoning_content = (
+ "\n".join(reasoning_segments) if reasoning_segments else None
+ )
+ return normalized_text, thinking_blocks, reasoning_content
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/embedding.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/embedding.py
new file mode 100644
index 0000000000000000000000000000000000000000..4861674a191fb8d5fb607a51525eefdbc7ec842f
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/embedding.py
@@ -0,0 +1,4 @@
+"""
+Calls handled in openai/
+as mistral is an openai-compatible endpoint.
+"""
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/mistral_embedding_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/mistral_embedding_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..54eed416c1e796020ca3df022b8362ac0ad853a1
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/__init__.py
@@ -0,0 +1 @@
+"""Mistral OCR transformation module."""
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/guardrail_translation/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/guardrail_translation/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..da7b6ee6bf09b0db1968ab3818d61766ff2e837e
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/guardrail_translation/__init__.py
@@ -0,0 +1,11 @@
+"""Mistral OCR handler for Unified Guardrails."""
+
+from litellm.llms.mistral.ocr.guardrail_translation.handler import OCRHandler
+from litellm.types.utils import CallTypes
+
+guardrail_translation_mappings = {
+ CallTypes.ocr: OCRHandler,
+ CallTypes.aocr: OCRHandler,
+}
+
+__all__ = ["guardrail_translation_mappings", "OCRHandler"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/guardrail_translation/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/guardrail_translation/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..697bd2daa3da9632124a667fd05168a1c3dabaf5
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/guardrail_translation/handler.py
@@ -0,0 +1,153 @@
+"""
+OCR Handler for Unified Guardrails
+
+Provides guardrail translation support for the OCR endpoint.
+Processes the extracted markdown text from OCR pages.
+"""
+
+from typing import TYPE_CHECKING, Any, List, Optional
+
+from litellm._logging import verbose_proxy_logger
+from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
+from litellm.types.utils import GenericGuardrailAPIInputs
+
+if TYPE_CHECKING:
+ from litellm.integrations.custom_guardrail import CustomGuardrail
+ from litellm.llms.base_llm.ocr.transformation import OCRResponse
+
+
+class OCRHandler(BaseTranslation):
+ """
+ Handler for processing OCR requests/responses with guardrails.
+
+ Input: The OCR input is a document URL/reference - not text content.
+ We pass the document URL as text for guardrails that may want to
+ validate or filter document sources.
+
+ Output: OCR responses contain extracted markdown text per page.
+ The handler extracts all page markdown, applies guardrails,
+ and maps the guardrailed text back to the pages.
+ """
+
+ async def process_input_messages(
+ self,
+ data: dict,
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process OCR input by applying guardrails to the document reference.
+
+ The OCR input contains a document dict with a URL. We extract
+ the URL and pass it to the guardrail for validation.
+
+ Args:
+ data: Request data containing 'document' parameter
+ guardrail_to_apply: The guardrail instance to apply
+ litellm_logging_obj: Optional logging object
+
+ Returns:
+ Modified data with guardrails applied
+ """
+ document = data.get("document")
+ if document is None or not isinstance(document, dict):
+ verbose_proxy_logger.debug(
+ "OCR guardrail: No valid document found in request data"
+ )
+ return data
+
+ # Extract the document URL for guardrail checking
+ texts_to_check: List[str] = []
+ doc_type = document.get("type")
+ if doc_type == "document_url":
+ url = document.get("document_url")
+ if url and isinstance(url, str):
+ texts_to_check.append(url)
+ elif doc_type == "image_url":
+ url = document.get("image_url")
+ if url and isinstance(url, str):
+ texts_to_check.append(url)
+
+ if not texts_to_check:
+ return data
+
+ inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
+ model = data.get("model")
+ if model:
+ inputs["model"] = model
+
+ await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=data,
+ input_type="request",
+ logging_obj=litellm_logging_obj,
+ )
+
+ return data
+
+ async def process_output_response(
+ self,
+ response: "OCRResponse",
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ user_api_key_dict: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process OCR output by applying guardrails to extracted page text.
+
+ Extracts markdown text from each OCR page, applies guardrails,
+ and maps the guardrailed text back to the pages.
+
+ Args:
+ response: OCRResponse with pages containing markdown text
+ guardrail_to_apply: The guardrail instance to apply
+ litellm_logging_obj: Optional logging object
+ user_api_key_dict: User API key metadata
+
+ Returns:
+ Modified OCRResponse with guardrailed page text
+ """
+ if not hasattr(response, "pages") or not response.pages:
+ verbose_proxy_logger.debug("OCR guardrail: No pages found in OCR response")
+ return response
+
+ # Extract markdown text from all pages
+ texts_to_check: List[str] = []
+ page_indices: List[int] = []
+ for i, page in enumerate(response.pages):
+ if hasattr(page, "markdown") and page.markdown:
+ texts_to_check.append(page.markdown)
+ page_indices.append(i)
+
+ if not texts_to_check:
+ return response
+
+ inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
+ model = getattr(response, "model", None)
+ if model:
+ inputs["model"] = model
+
+ # Add user metadata if available
+ if user_api_key_dict is not None:
+ metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
+ inputs.update(metadata) # type: ignore
+
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data={},
+ input_type="response",
+ logging_obj=litellm_logging_obj,
+ )
+
+ # Map guardrailed text back to pages
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+ for idx, page_idx in enumerate(page_indices):
+ if idx < len(guardrailed_texts):
+ response.pages[page_idx].markdown = guardrailed_texts[idx]
+
+ verbose_proxy_logger.debug(
+ "OCR guardrail: Applied guardrail to %d pages",
+ len(guardrailed_texts),
+ )
+
+ return response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..3d5e876302737eb8eb9e94464cbc08b74b903d9d
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/transformation.py
@@ -0,0 +1,225 @@
+"""
+Mistral OCR transformation implementation.
+"""
+from typing import Any, Dict, Optional
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.llms.base_llm.ocr.transformation import (
+ BaseOCRConfig,
+ DocumentType,
+ OCRRequestData,
+ OCRResponse,
+)
+from litellm.secret_managers.main import get_secret_str
+
+
+class MistralOCRConfig(BaseOCRConfig):
+ """
+ Mistral OCR transformation configuration.
+
+ Reference: https://docs.mistral.ai/api/#tag/ocr
+ """
+
+ def __init__(self) -> None:
+ super().__init__()
+
+ def get_supported_ocr_params(self, model: str) -> list:
+ """
+ Get supported OCR parameters for Mistral OCR.
+
+ Mistral OCR supports:
+ - pages: List of page numbers to process
+ - include_image_base64: Whether to include base64 encoded images
+ - image_limit: Maximum number of images to return
+ - image_min_size: Minimum size of images to include
+ - bbox_annotation_format: Format for bounding box annotations
+ - document_annotation_format: Format for document annotations
+ - extract_header: Whether to extract document header
+ - extract_footer: Whether to extract document footer
+ """
+ return [
+ "pages",
+ "include_image_base64",
+ "image_limit",
+ "image_min_size",
+ "bbox_annotation_format",
+ "document_annotation_format",
+ "extract_header",
+ "extract_footer",
+ ]
+
+ def map_ocr_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ ) -> dict:
+ """
+ Map OCR parameters to Mistral-specific format.
+
+ Mistral accepts these parameters directly, so no transformation needed.
+ Just filter out unsupported params.
+ """
+ supported_params = self.get_supported_ocr_params(model=model)
+
+ # Only include params that are in the supported list
+ mapped_params = {}
+ for param, value in non_default_params.items():
+ if param in supported_params:
+ mapped_params[param] = value
+
+ return mapped_params
+
+ def validate_environment(
+ self,
+ headers: Dict,
+ model: str,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ litellm_params: Optional[dict] = None,
+ **kwargs,
+ ) -> Dict:
+ """
+ Validate environment and return headers for Mistral OCR.
+ """
+ # Get API key from environment if not provided
+ if api_key is None:
+ api_key = get_secret_str("MISTRAL_API_KEY")
+
+ if api_key is None:
+ raise ValueError(
+ "Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params"
+ )
+
+ headers = {
+ "Authorization": f"Bearer {api_key}",
+ **headers,
+ }
+
+ # Don't set Content-Type for multipart/form-data - httpx will handle it
+
+ return headers
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: Optional[dict] = None,
+ **kwargs,
+ ) -> str:
+ """
+ Get complete URL for Mistral OCR endpoint.
+
+ Returns: https://api.mistral.ai/v1/ocr
+ """
+ if api_base is None:
+ api_base = "https://api.mistral.ai/v1"
+
+ # Ensure no trailing slash
+ api_base = api_base.rstrip("/")
+
+ # Remove /v1 if it's already in the base to avoid duplication
+ if api_base.endswith("/v1"):
+ return f"{api_base}/ocr"
+
+ return f"{api_base}/v1/ocr"
+
+ def transform_ocr_request(
+ self,
+ model: str,
+ document: DocumentType,
+ optional_params: dict,
+ headers: dict,
+ **kwargs,
+ ) -> OCRRequestData:
+ """
+ Transform OCR request to Mistral-specific format.
+
+ Mistral OCR API accepts:
+ {
+ "model": "mistral-ocr-latest",
+ "document": {
+ "type": "document_url",
+ "document_url": ""
+ },
+ "pages": [0], # optional
+ "include_image_base64": false, # optional
+ ...
+ }
+
+ Args:
+ model: Model name (e.g., "mistral-ocr-latest")
+ document: Document dict from user (Mistral format) - already validated in main.py
+ optional_params: Already mapped optional parameters
+ headers: Request headers
+
+ Returns:
+ OCRRequestData with JSON data
+ """
+ verbose_logger.debug(f"Mistral OCR transform_ocr_request - model: {model}")
+
+ # Document parameter is the Mistral-format dict from the user
+ # Just pass it through as-is to the Mistral API
+ if not isinstance(document, dict):
+ raise ValueError(f"Expected document dict, got {type(document)}")
+
+ # Build request data - use document dict directly
+ data = {
+ "model": model,
+ "document": document, # Pass through the Mistral-format document dict
+ }
+
+ # Add all optional parameters from the already-mapped optional_params
+ data.update(optional_params)
+
+ # No multipart files - using JSON
+ return OCRRequestData(data=data, files=None)
+
+ def transform_ocr_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ logging_obj: Any,
+ **kwargs,
+ ) -> OCRResponse:
+ """
+ Return Mistral OCR response in native format.
+
+ Mistral OCR is the standard format for LiteLLM OCR responses.
+ No transformation needed - return native response.
+
+ Mistral OCR returns:
+ {
+ "pages": [
+ {
+ "index": 0,
+ "markdown": "extracted text content",
+ "images": [...],
+ "dimensions": {...}
+ },
+ ...
+ ],
+ "model": "mistral-ocr-2505-completion",
+ "document_annotation": null,
+ "usage_info": {...}
+ }
+ """
+ try:
+ response_json = raw_response.json()
+
+ verbose_logger.debug(f"Mistral OCR response keys: {response_json.keys()}")
+
+ # Return native Mistral format - no transformation
+ return OCRResponse(
+ pages=response_json.get("pages", []),
+ model=response_json.get("model", model),
+ document_annotation=response_json.get("document_annotation"),
+ usage_info=response_json.get("usage_info"),
+ object="ocr",
+ )
+ except Exception as e:
+ verbose_logger.error(f"Error parsing Mistral OCR response: {e}")
+ raise e
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/moonshot/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/moonshot/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..e4d7b5f033b051421c00f188e5085e6207514c12
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/moonshot/chat/transformation.py
@@ -0,0 +1,246 @@
+"""
+Translates from OpenAI's `/v1/chat/completions` to Moonshot AI's `/v1/chat/completions`
+"""
+
+from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
+
+import litellm
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+ handle_messages_with_content_list_to_str_conversion,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import AllMessageValues
+from litellm.utils import supports_reasoning
+
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+
+
+class MoonshotChatConfig(OpenAIGPTConfig):
+ @overload
+ def _transform_messages(
+ self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
+ ) -> Coroutine[Any, Any, List[AllMessageValues]]:
+ ...
+
+ @overload
+ def _transform_messages(
+ self,
+ messages: List[AllMessageValues],
+ model: str,
+ is_async: Literal[False] = False,
+ ) -> List[AllMessageValues]:
+ ...
+
+ def _transform_messages(
+ self, messages: List[AllMessageValues], model: str, is_async: bool = False
+ ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
+ """
+ Moonshot text-only models don't support content in list format.
+ Multimodal models (kimi-k2.5, kimi-latest, etc.) accept the
+ standard OpenAI content array with non-text blocks (image_url,
+ input_audio, video_url, file, etc.).
+
+ If any message contains a non-text content part, skip flattening
+ so the multimodal payload is preserved.
+ """
+ has_non_text = False
+ for m in messages:
+ _content = m.get("content")
+ if _content and isinstance(_content, list):
+ if any(c.get("type") != "text" for c in _content):
+ has_non_text = True
+ break
+
+ if not has_non_text:
+ messages = handle_messages_with_content_list_to_str_conversion(messages)
+
+ if is_async:
+ return super()._transform_messages(
+ messages=messages, model=model, is_async=True
+ )
+ else:
+ return super()._transform_messages(
+ messages=messages, model=model, is_async=False
+ )
+
+ def _get_openai_compatible_provider_info(
+ self, api_base: Optional[str], api_key: Optional[str]
+ ) -> Tuple[Optional[str], Optional[str]]:
+ api_base = api_base or get_secret_str("MOONSHOT_API_BASE") or "https://api.moonshot.ai/v1" # type: ignore
+ dynamic_api_key = api_key or get_secret_str("MOONSHOT_API_KEY")
+ return api_base, dynamic_api_key
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ api_key: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: dict,
+ stream: Optional[bool] = None,
+ ) -> str:
+ """
+ If api_base is not provided, use the default Moonshot AI /chat/completions endpoint.
+ """
+ if not api_base:
+ api_base = "https://api.moonshot.ai/v1"
+
+ if not api_base.endswith("/chat/completions"):
+ api_base = f"{api_base}/chat/completions"
+
+ return api_base
+
+ def get_supported_openai_params(self, model: str) -> list:
+ """
+ Get the supported OpenAI params for Moonshot AI models
+
+ Moonshot AI limitations:
+ - functions parameter is not supported (use tools instead)
+ - tool_choice doesn't support "required" value
+ - kimi-thinking-preview doesn't support tool calls at all
+ """
+ excluded_params: List[str] = ["functions"]
+
+ # kimi-thinking-preview has additional limitations
+ if "kimi-thinking-preview" in model:
+ excluded_params.extend(["tools", "tool_choice"])
+
+ base_openai_params = super().get_supported_openai_params(model=model)
+ final_params: List[str] = []
+ for param in base_openai_params:
+ if param not in excluded_params:
+ final_params.append(param)
+
+ return final_params
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ """
+ Map OpenAI parameters to Moonshot AI parameters
+
+ Handles Moonshot AI specific limitations:
+ - tool_choice doesn't support "required" value
+ - Temperature <0.3 limitation for n>1
+ """
+ supported_openai_params = self.get_supported_openai_params(model)
+ for param, value in non_default_params.items():
+ if param == "max_completion_tokens":
+ optional_params["max_tokens"] = value
+ elif param in supported_openai_params:
+ optional_params[param] = value
+
+ ##########################################
+ # temperature limitations
+ # 1. `temperature` on KIMI API is [0, 1] but OpenAI is [0, 2]
+ # 2. If temperature < 0.3 and n > 1, KIMI will raise an exception.
+ # If we enter this condition, we set the temperature to 0.3 as suggested by Moonshot AI
+ ##########################################
+ if "temperature" in optional_params:
+ if optional_params["temperature"] > 1:
+ optional_params["temperature"] = 1
+ if optional_params["temperature"] < 0.3 and optional_params.get("n", 1) > 1:
+ optional_params["temperature"] = 0.3
+ return optional_params
+
+ def fill_reasoning_content(
+ self, messages: List[AllMessageValues]
+ ) -> List[AllMessageValues]:
+ """
+ Moonshot reasoning models require `reasoning_content` on every assistant
+ message that contains tool_calls (multi-turn tool-calling flows).
+
+ For each such message that is missing the field:
+ 1. Check if reasoning_content exists at the top level (for Pydantic models
+ that have the attribute but don't support 'in' operator)
+ 2. Promote provider_specific_fields["reasoning_content"] if present and non-empty
+ (this is where LiteLLM stores it from a previous response)
+ 3. Otherwise inject a single space — the minimum value the API accepts
+ Messages that already carry the field, or are not assistant/tool-call messages,
+ are appended as-is (no copy made).
+ """
+ result: List[AllMessageValues] = []
+ for msg in messages:
+ if (
+ msg.get("role") == "assistant"
+ and msg.get("tool_calls")
+ and not msg.get(
+ "reasoning_content"
+ ) # Check using .get() which works for both dicts and Pydantic models
+ ):
+ patched = dict(cast(dict, msg))
+ provider_fields = patched.get("provider_specific_fields") or {}
+ stored = provider_fields.get("reasoning_content")
+ if stored:
+ patched["reasoning_content"] = stored
+ # Remove the promoted key from provider_specific_fields to
+ # avoid sending the value twice in the serialised request body
+ cleaned_provider_fields = dict(provider_fields)
+ cleaned_provider_fields.pop("reasoning_content", None)
+ patched["provider_specific_fields"] = cleaned_provider_fields
+ else:
+ litellm.verbose_logger.warning(
+ "Moonshot reasoning model: assistant tool-call message is missing "
+ "`reasoning_content`. Injecting a placeholder to satisfy API validation. "
+ "For best results, preserve `reasoning_content` from the original "
+ "assistant response when building multi-turn conversation history."
+ )
+ patched["reasoning_content"] = " "
+ result.append(cast(AllMessageValues, patched))
+ else:
+ result.append(msg)
+ return result
+
+ def transform_request(
+ self,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ headers: dict,
+ ) -> dict:
+ """
+ Transform the overall request to be sent to the API.
+ Returns:
+ dict: The transformed request. Sent as the body of the API call.
+ """
+ # Add tool_choice="required" message if needed
+ if optional_params.get("tool_choice", None) == "required":
+ messages = self._add_tool_choice_required_message(
+ messages=messages,
+ optional_params=optional_params,
+ )
+
+ # Moonshot reasoning models: fill in reasoning_content before the API call
+ if supports_reasoning(model=model, custom_llm_provider="moonshot"):
+ messages = self.fill_reasoning_content(messages)
+
+ # Call parent transform_request which handles _transform_messages
+ return super().transform_request(
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ headers=headers,
+ )
+
+ def _add_tool_choice_required_message(
+ self, messages: List[AllMessageValues], optional_params: dict
+ ) -> List[AllMessageValues]:
+ """
+ Add a message to the messages list to indicate that the tool choice is required.
+
+ https://platform.moonshot.ai/docs/guide/migrating-from-openai-to-kimi#about-tool_choice
+ """
+ messages.append(
+ {
+ "role": "user",
+ "content": "Please select a tool to handle the current issue.", # Usually, the Kimi large language model understands the intention to invoke a tool and selects one for invocation
+ }
+ )
+ optional_params.pop("tool_choice")
+ return messages
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/chat/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/chat/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..93bd7e16aef73d0caf6835ff708d37cf2c458e86
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/chat/transformation.py
@@ -0,0 +1,40 @@
+"""
+Transform request from OpenAI format to Morph format.
+
+[TODO] Docs: Morph supports the OpenAI API format.
+https://docs.morphllm.com/quickstart
+"""
+
+from typing import Optional, Tuple
+
+from litellm.secret_managers.main import get_secret_str
+
+from ...openai_like.chat.transformation import OpenAILikeChatConfig
+
+
+class MorphChatConfig(OpenAILikeChatConfig):
+ """
+ Transform request from OpenAI format to Morph format.
+ """
+
+ @property
+ def custom_llm_provider(self) -> Optional[str]:
+ return "morph"
+
+ def _get_openai_compatible_provider_info(
+ self, api_base: Optional[str], api_key: Optional[str]
+ ) -> Tuple[Optional[str], Optional[str]]:
+ api_base = (
+ api_base
+ or get_secret_str("MORPH_API_BASE")
+ or "https://api.morphllm.com/v1" # default api base
+ )
+ dynamic_api_key = api_key or get_secret_str("MORPH_API_KEY")
+ return api_base, dynamic_api_key
+
+ def get_supported_openai_params(self, model: str) -> list:
+ return [
+ "messages",
+ "model",
+ "stream",
+ ]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nebius/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nebius/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..cb713147718f484751a911209bd3a0862127708b
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nebius/chat/transformation.py
@@ -0,0 +1,27 @@
+"""
+Nebius AI Studio Chat Completions API - Transformation
+
+This is OpenAI compatible - no translation needed / occurs
+"""
+
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+
+
+class NebiusConfig(OpenAIGPTConfig):
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ """
+ map max_completion_tokens param to max_tokens
+ """
+ supported_openai_params = self.get_supported_openai_params(model=model)
+ for param, value in non_default_params.items():
+ if param == "max_completion_tokens":
+ optional_params["max_tokens"] = value
+ elif param in supported_openai_params:
+ optional_params[param] = value
+ return optional_params
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nebius/embedding/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nebius/embedding/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..d56b7def13cc15f2c2133eb6c92ef73db0408d79
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nebius/embedding/transformation.py
@@ -0,0 +1,5 @@
+"""
+Calls handled in openai/
+
+as Nebius AI Studio is an openai-compatible endpoint.
+"""
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/chat/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/chat/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0563d8b5532a386108cd991aeb6e8feb49e6aea
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/chat/handler.py
@@ -0,0 +1,131 @@
+import json
+from typing import Callable, Optional, Union
+
+import litellm
+from litellm.llms.custom_httpx.http_handler import (
+ AsyncHTTPHandler,
+ HTTPHandler,
+ _get_httpx_client,
+)
+from litellm.utils import ModelResponse
+
+from .transformation import NLPCloudConfig
+
+nlp_config = NLPCloudConfig()
+
+
+def completion(
+ model: str,
+ messages: list,
+ api_base: str,
+ model_response: ModelResponse,
+ print_verbose: Callable,
+ encoding,
+ api_key,
+ logging_obj,
+ optional_params: dict,
+ litellm_params: dict,
+ logger_fn=None,
+ default_max_tokens_to_sample=None,
+ client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
+ headers={},
+):
+ headers = nlp_config.validate_environment(
+ api_key=api_key,
+ headers=headers,
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ )
+
+ ## Load Config
+ config = litellm.NLPCloudConfig.get_config()
+ for k, v in config.items():
+ if (
+ k not in optional_params
+ ): # completion(top_k=3) > togetherai_config(top_k=3) <- allows for dynamic variables to be passed in
+ optional_params[k] = v
+
+ completion_url_fragment_1 = api_base
+ completion_url_fragment_2 = "/generation"
+ model = model
+
+ completion_url = completion_url_fragment_1 + model + completion_url_fragment_2
+ data = nlp_config.transform_request(
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ headers=headers,
+ )
+
+ ## LOGGING
+ logging_obj.pre_call(
+ input=None,
+ api_key=api_key,
+ additional_args={
+ "complete_input_dict": data,
+ "headers": headers,
+ "api_base": completion_url,
+ },
+ )
+ ## COMPLETION CALL
+ if client is None or not isinstance(client, HTTPHandler):
+ client = _get_httpx_client()
+
+ response = client.post(
+ completion_url,
+ headers=headers,
+ data=json.dumps(data),
+ stream=optional_params["stream"] if "stream" in optional_params else False,
+ )
+ if "stream" in optional_params and optional_params["stream"] is True:
+ return clean_and_iterate_chunks(response)
+ else:
+ return nlp_config.transform_response(
+ model=model,
+ raw_response=response,
+ model_response=model_response,
+ logging_obj=logging_obj,
+ api_key=api_key,
+ request_data=data,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ encoding=encoding,
+ )
+
+
+# def clean_and_iterate_chunks(response):
+# def process_chunk(chunk):
+# print(f"received chunk: {chunk}")
+# cleaned_chunk = chunk.decode("utf-8")
+# # Perform further processing based on your needs
+# return cleaned_chunk
+
+
+# for line in response.iter_lines():
+# if line:
+# yield process_chunk(line)
+def clean_and_iterate_chunks(response):
+ buffer = b""
+
+ for chunk in response.iter_content(chunk_size=1024):
+ if not chunk:
+ break
+
+ buffer += chunk
+ while b"\x00" in buffer:
+ buffer = buffer.replace(b"\x00", b"")
+ yield buffer.decode("utf-8")
+ buffer = b""
+
+ # No more data expected, yield any remaining data in the buffer
+ if buffer:
+ yield buffer.decode("utf-8")
+
+
+def embedding():
+ # logic for parsing in - calling - parsing out model embedding calls
+ pass
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..8037a458321ea3198f504b3a5ff25dfdd2e57dc4
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/chat/transformation.py
@@ -0,0 +1,228 @@
+import json
+import time
+from typing import TYPE_CHECKING, Any, List, Optional, Union
+
+import httpx
+
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+ convert_content_list_to_str,
+)
+from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
+from litellm.types.llms.openai import AllMessageValues
+from litellm.utils import ModelResponse, Usage
+
+from ..common_utils import NLPCloudError
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+ LoggingClass = LiteLLMLoggingObj
+else:
+ LoggingClass = Any
+
+
+class NLPCloudConfig(BaseConfig):
+ """
+ Reference: https://docs.nlpcloud.com/#generation
+
+ - `max_length` (int): Optional. The maximum number of tokens that the generated text should contain.
+
+ - `length_no_input` (boolean): Optional. Whether `min_length` and `max_length` should not include the length of the input text.
+
+ - `end_sequence` (string): Optional. A specific token that should be the end of the generated sequence.
+
+ - `remove_end_sequence` (boolean): Optional. Whether to remove the `end_sequence` string from the result.
+
+ - `remove_input` (boolean): Optional. Whether to remove the input text from the result.
+
+ - `bad_words` (list of strings): Optional. List of tokens that are not allowed to be generated.
+
+ - `temperature` (float): Optional. Temperature sampling. It modulates the next token probabilities.
+
+ - `top_p` (float): Optional. Top P sampling. Below 1, only the most probable tokens with probabilities that add up to top_p or higher are kept for generation.
+
+ - `top_k` (int): Optional. Top K sampling. The number of highest probability vocabulary tokens to keep for top k filtering.
+
+ - `repetition_penalty` (float): Optional. Prevents the same word from being repeated too many times.
+
+ - `num_beams` (int): Optional. Number of beams for beam search.
+
+ - `num_return_sequences` (int): Optional. The number of independently computed returned sequences.
+ """
+
+ max_length: Optional[int] = None
+ length_no_input: Optional[bool] = None
+ end_sequence: Optional[str] = None
+ remove_end_sequence: Optional[bool] = None
+ remove_input: Optional[bool] = None
+ bad_words: Optional[list] = None
+ temperature: Optional[float] = None
+ top_p: Optional[float] = None
+ top_k: Optional[int] = None
+ repetition_penalty: Optional[float] = None
+ num_beams: Optional[int] = None
+ num_return_sequences: Optional[int] = None
+
+ def __init__(
+ self,
+ max_length: Optional[int] = None,
+ length_no_input: Optional[bool] = None,
+ end_sequence: Optional[str] = None,
+ remove_end_sequence: Optional[bool] = None,
+ remove_input: Optional[bool] = None,
+ bad_words: Optional[list] = None,
+ temperature: Optional[float] = None,
+ top_p: Optional[float] = None,
+ top_k: Optional[int] = None,
+ repetition_penalty: Optional[float] = None,
+ num_beams: Optional[int] = None,
+ num_return_sequences: Optional[int] = None,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return super().get_config()
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ headers = {
+ "accept": "application/json",
+ "content-type": "application/json",
+ }
+ if api_key:
+ headers["Authorization"] = f"Token {api_key}"
+ return headers
+
+ def get_supported_openai_params(self, model: str) -> List:
+ return [
+ "max_tokens",
+ "stream",
+ "temperature",
+ "top_p",
+ "presence_penalty",
+ "frequency_penalty",
+ "n",
+ "stop",
+ ]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ for param, value in non_default_params.items():
+ if param == "max_tokens":
+ optional_params["max_length"] = value
+ if param == "stream":
+ optional_params["stream"] = value
+ if param == "temperature":
+ optional_params["temperature"] = value
+ if param == "top_p":
+ optional_params["top_p"] = value
+ if param == "presence_penalty":
+ optional_params["presence_penalty"] = value
+ if param == "frequency_penalty":
+ optional_params["frequency_penalty"] = value
+ if param == "n":
+ optional_params["num_return_sequences"] = value
+ if param == "stop":
+ optional_params["stop_sequences"] = value
+ return optional_params
+
+ def get_error_class(
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+ ) -> BaseLLMException:
+ return NLPCloudError(
+ status_code=status_code, message=error_message, headers=headers
+ )
+
+ def transform_request(
+ self,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ headers: dict,
+ ) -> dict:
+ text = " ".join(convert_content_list_to_str(message) for message in messages)
+
+ data = {
+ "text": text,
+ **optional_params,
+ }
+
+ return data
+
+ def transform_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ model_response: ModelResponse,
+ logging_obj: LoggingClass,
+ request_data: dict,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: Any,
+ api_key: Optional[str] = None,
+ json_mode: Optional[bool] = None,
+ ) -> ModelResponse:
+ ## LOGGING
+ logging_obj.post_call(
+ input=None,
+ api_key=api_key,
+ original_response=raw_response.text,
+ additional_args={"complete_input_dict": request_data},
+ )
+
+ ## RESPONSE OBJECT
+ try:
+ completion_response = raw_response.json()
+ except Exception:
+ raise NLPCloudError(
+ message=raw_response.text, status_code=raw_response.status_code
+ )
+ if "error" in completion_response:
+ raise NLPCloudError(
+ message=completion_response["error"],
+ status_code=raw_response.status_code,
+ )
+ else:
+ try:
+ if len(completion_response["generated_text"]) > 0:
+ model_response.choices[0].message.content = ( # type: ignore
+ completion_response["generated_text"]
+ )
+ except Exception:
+ raise NLPCloudError(
+ message=json.dumps(completion_response),
+ status_code=raw_response.status_code,
+ )
+
+ ## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
+ prompt_tokens = completion_response["nb_input_tokens"]
+ completion_tokens = completion_response["nb_generated_tokens"]
+
+ model_response.created = int(time.time())
+ model_response.model = model
+ usage = Usage(
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ total_tokens=prompt_tokens + completion_tokens,
+ )
+ setattr(model_response, "usage", usage)
+ return model_response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/common_utils.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/common_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..232f56c970941047b3d7e79b002aa19072d6091e
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/common_utils.py
@@ -0,0 +1,15 @@
+from typing import Optional, Union
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+
+class NLPCloudError(BaseLLMException):
+ def __init__(
+ self,
+ status_code: int,
+ message: str,
+ headers: Optional[Union[dict, httpx.Headers]] = None,
+ ):
+ super().__init__(status_code=status_code, message=message, headers=headers)
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/novita/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/novita/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..c05d2d7b2c5a66598acff3cecab74ce0e404ab34
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/novita/chat/transformation.py
@@ -0,0 +1,33 @@
+"""
+Support for OpenAI's `/v1/chat/completions` endpoint.
+
+Calls done in OpenAI/openai.py as Novita AI is openai-compatible.
+
+Docs: https://novita.ai/docs/guides/llm-api
+"""
+
+from typing import List, Optional
+
+from ....types.llms.openai import AllMessageValues
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+
+
+class NovitaConfig(OpenAIGPTConfig):
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ if api_key is None:
+ raise ValueError(
+ "Missing Novita AI API Key - A call is being made to novita but no key is set either in the environment variables or via params"
+ )
+ headers["Authorization"] = f"Bearer {api_key}"
+ headers["Content-Type"] = "application/json"
+ headers["X-Novita-Source"] = "litellm"
+ return headers
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nscale/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nscale/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..6103b8e3c498c983d18faae89a652d3523b9926b
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nscale/chat/transformation.py
@@ -0,0 +1,52 @@
+from typing import Optional, Tuple
+
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+from litellm.secret_managers.main import get_secret_str
+
+
+class NscaleConfig(OpenAIGPTConfig):
+ """
+ Reference: Nscale is OpenAI compatible.
+ API Key: NSCALE_API_KEY
+ Default API Base: https://inference.api.nscale.com/v1
+ """
+
+ API_BASE_URL = "https://inference.api.nscale.com/v1"
+
+ @property
+ def custom_llm_provider(self) -> Optional[str]:
+ return "nscale"
+
+ @staticmethod
+ def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
+ return api_key or get_secret_str("NSCALE_API_KEY")
+
+ @staticmethod
+ def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
+ return (
+ api_base or get_secret_str("NSCALE_API_BASE") or NscaleConfig.API_BASE_URL
+ )
+
+ def _get_openai_compatible_provider_info(
+ self, api_base: Optional[str], api_key: Optional[str]
+ ) -> Tuple[Optional[str], Optional[str]]:
+ # This method is called by get_llm_provider to resolve api_base and api_key
+ resolved_api_base = NscaleConfig.get_api_base(api_base)
+ resolved_api_key = NscaleConfig.get_api_key(api_key)
+ return resolved_api_base, resolved_api_key
+
+ def get_supported_openai_params(self, model: str) -> list:
+ return [
+ "max_tokens",
+ "n",
+ "temperature",
+ "top_p",
+ "stream",
+ "logprobs",
+ "top_logprobs",
+ "frequency_penalty",
+ "presence_penalty",
+ "response_format",
+ "stop",
+ "logit_bias",
+ ]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..e687229949bbe014ffaa7d9234cd7984fb776d37
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/chat/transformation.py
@@ -0,0 +1,110 @@
+"""
+Nvidia NIM endpoint: https://docs.api.nvidia.com/nim/reference/databricks-dbrx-instruct-infer
+
+This is OpenAI compatible
+
+This file only contains param mapping logic
+
+API calling is done using the OpenAI SDK with an api_base
+"""
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+
+
+class NvidiaNimConfig(OpenAIGPTConfig):
+ """
+ Reference: https://docs.api.nvidia.com/nim/reference/databricks-dbrx-instruct-infer
+
+ The class `NvidiaNimConfig` provides configuration for the Nvidia NIM's Chat Completions API interface. Below are the parameters:
+ """
+
+ def get_supported_openai_params(self, model: str) -> list:
+ """
+ Get the supported OpenAI params for the given model
+
+
+ Updated on July 5th, 2024 - based on https://docs.api.nvidia.com/nim/reference
+ """
+ if model in [
+ "google/recurrentgemma-2b",
+ "google/gemma-2-27b-it",
+ "google/gemma-2-9b-it",
+ "gemma-2-9b-it",
+ ]:
+ return ["stream", "temperature", "top_p", "max_tokens", "stop", "seed"]
+ elif model == "nvidia/nemotron-4-340b-instruct":
+ return [
+ "stream",
+ "temperature",
+ "top_p",
+ "max_tokens",
+ "max_completion_tokens",
+ ]
+ elif model == "nvidia/nemotron-4-340b-reward":
+ return [
+ "stream",
+ ]
+ elif model in ["google/codegemma-1.1-7b"]:
+ # most params - but no 'seed' :(
+ return [
+ "stream",
+ "temperature",
+ "top_p",
+ "frequency_penalty",
+ "presence_penalty",
+ "max_tokens",
+ "max_completion_tokens",
+ "stop",
+ ]
+ else:
+ # DEFAULT Case - The vast majority of Nvidia NIM Models lie here
+ # "upstage/solar-10.7b-instruct",
+ # "snowflake/arctic",
+ # "seallms/seallm-7b-v2.5",
+ # "nvidia/llama3-chatqa-1.5-8b",
+ # "nvidia/llama3-chatqa-1.5-70b",
+ # "mistralai/mistral-large",
+ # "mistralai/mixtral-8x22b-instruct-v0.1",
+ # "mistralai/mixtral-8x7b-instruct-v0.1",
+ # "mistralai/mistral-7b-instruct-v0.3",
+ # "mistralai/mistral-7b-instruct-v0.2",
+ # "mistralai/codestral-22b-instruct-v0.1",
+ # "microsoft/phi-3-small-8k-instruct",
+ # "microsoft/phi-3-small-128k-instruct",
+ # "microsoft/phi-3-mini-4k-instruct",
+ # "microsoft/phi-3-mini-128k-instruct",
+ # "microsoft/phi-3-medium-4k-instruct",
+ # "microsoft/phi-3-medium-128k-instruct",
+ # "meta/llama3-70b-instruct",
+ # "meta/llama3-8b-instruct",
+ # "meta/llama2-70b",
+ # "meta/codellama-70b",
+ return [
+ "stream",
+ "temperature",
+ "top_p",
+ "frequency_penalty",
+ "presence_penalty",
+ "max_tokens",
+ "max_completion_tokens",
+ "stop",
+ "seed",
+ "tools",
+ "tool_choice",
+ "parallel_tool_calls",
+ "response_format",
+ ]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ supported_openai_params = self.get_supported_openai_params(model=model)
+ for param, value in non_default_params.items():
+ if param == "max_completion_tokens":
+ optional_params["max_tokens"] = value
+ elif param in supported_openai_params:
+ optional_params[param] = value
+ return optional_params
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/embed.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/embed.py
new file mode 100644
index 0000000000000000000000000000000000000000..24c6cc34e4d6aff7a6fd867ac45778bba16411b8
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/embed.py
@@ -0,0 +1,82 @@
+"""
+Nvidia NIM embeddings endpoint: https://docs.api.nvidia.com/nim/reference/nvidia-nv-embedqa-e5-v5-infer
+
+This is OpenAI compatible
+
+This file only contains param mapping logic
+
+API calling is done using the OpenAI SDK with an api_base
+"""
+
+import types
+from typing import Optional
+
+
+class NvidiaNimEmbeddingConfig:
+ """
+ Reference: https://docs.api.nvidia.com/nim/reference/nvidia-nv-embedqa-e5-v5-infer
+ """
+
+ # OpenAI params
+ encoding_format: Optional[str] = None
+ user: Optional[str] = None
+
+ # Nvidia NIM params
+ input_type: Optional[str] = None
+ truncate: Optional[str] = None
+
+ def __init__(
+ self,
+ encoding_format: Optional[str] = None,
+ user: Optional[str] = None,
+ input_type: Optional[str] = None,
+ truncate: Optional[str] = None,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return {
+ k: v
+ for k, v in cls.__dict__.items()
+ if not k.startswith("__")
+ and not isinstance(
+ v,
+ (
+ types.FunctionType,
+ types.BuiltinFunctionType,
+ classmethod,
+ staticmethod,
+ ),
+ )
+ and v is not None
+ }
+
+ def get_supported_openai_params(
+ self,
+ ):
+ return ["encoding_format", "user", "dimensions"]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ kwargs: Optional[dict] = None,
+ ):
+ if "extra_body" not in optional_params:
+ optional_params["extra_body"] = {}
+ for k, v in non_default_params.items():
+ if k == "input_type":
+ optional_params["extra_body"].update({"input_type": v})
+ elif k == "truncate":
+ optional_params["extra_body"].update({"truncate": v})
+ else:
+ optional_params[k] = v
+
+ if kwargs is not None:
+ # pass kwargs in extra_body
+ optional_params["extra_body"].update(kwargs)
+ return optional_params
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/rerank/common_utils.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/rerank/common_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..738fb09364ac9e6638278e7b717b7f7dc802e856
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/rerank/common_utils.py
@@ -0,0 +1,27 @@
+"""
+Common utilities for NVIDIA NIM rerank provider.
+"""
+
+
+def get_nvidia_nim_rerank_config(model: str):
+ """
+ Get the appropriate NVIDIA NIM rerank config based on the model.
+
+ Args:
+ model: The model string (e.g., "nvidia/llama-3.2-nv-rerankqa-1b-v2" or "ranking/nvidia/llama-3.2-nv-rerankqa-1b-v2")
+
+ Returns:
+ NvidiaNimRankingConfig if model starts with "ranking/", else NvidiaNimRerankConfig
+
+ Example:
+ - "ranking/nvidia/llama-3.2-nv-rerankqa-1b-v2" -> NvidiaNimRankingConfig
+ - "nvidia/llama-3.2-nv-rerankqa-1b-v2" -> NvidiaNimRerankConfig
+ """
+ from litellm.llms.nvidia_nim.rerank.ranking_transformation import (
+ NvidiaNimRankingConfig,
+ )
+ from litellm.llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig
+
+ if model.startswith("ranking/"):
+ return NvidiaNimRankingConfig()
+ return NvidiaNimRerankConfig()
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/rerank/ranking_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/rerank/ranking_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..757d874bf311f31f04a8c9525bb1b77fee7d7cb0
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/rerank/ranking_transformation.py
@@ -0,0 +1,78 @@
+"""
+Transformation for NVIDIA NIM Ranking models that use /v1/ranking endpoint.
+
+Use this by passing "nvidia_nim/ranking/" to force the /v1/ranking endpoint.
+
+Reference: https://build.nvidia.com/nvidia/llama-3_2-nv-rerankqa-1b-v2/deploy
+"""
+
+from typing import Dict, Optional
+
+from litellm.llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig
+
+
+class NvidiaNimRankingConfig(NvidiaNimRerankConfig):
+ """
+ Configuration for NVIDIA NIM models that use the /v1/ranking endpoint.
+
+ Example:
+ curl -X "POST" 'https://ai.api.nvidia.com/v1/ranking' \
+ -H 'Accept: application/json' \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "model": "nvidia/llama-3.2-nv-rerankqa-1b-v2",
+ "query": {"text": "which way did the traveler go?"},
+ "passages": [{"text": "..."}, {"text": "..."}],
+ "truncate": "END"
+ }'
+ """
+
+ def _get_clean_model_name(self, model: str) -> str:
+ """Strip 'nvidia_nim/' and 'ranking/' prefixes from model name."""
+ # First strip nvidia_nim/ prefix if present
+ if model.startswith("nvidia_nim/"):
+ model = model[len("nvidia_nim/") :]
+ # Then strip ranking/ prefix if present
+ if model.startswith("ranking/"):
+ model = model[len("ranking/") :]
+ return model
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ model: str,
+ optional_params: Optional[dict] = None,
+ ) -> str:
+ """
+ Construct the Nvidia NIM ranking URL.
+
+ Format: {api_base}/v1/ranking
+ """
+ if not api_base:
+ api_base = self.DEFAULT_NIM_RERANK_API_BASE
+
+ api_base = api_base.rstrip("/")
+
+ if api_base.endswith("/ranking"):
+ return api_base
+
+ if api_base.endswith("/v1"):
+ api_base = api_base[:-3]
+
+ return f"{api_base}/v1/ranking"
+
+ def transform_rerank_request(
+ self,
+ model: str,
+ optional_rerank_params: Dict,
+ headers: dict,
+ ) -> dict:
+ """
+ Transform request, using clean model name without 'ranking/' prefix.
+ """
+ clean_model = self._get_clean_model_name(model)
+ return super().transform_rerank_request(
+ model=clean_model,
+ optional_rerank_params=optional_rerank_params,
+ headers=headers,
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/rerank/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/rerank/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..bd5abac60c839299ac554dbc19dc3c5e8fe15b2e
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nvidia_nim/rerank/transformation.py
@@ -0,0 +1,341 @@
+from typing import Any, Dict, List, Literal, Optional, Union
+
+import httpx
+from typing_extensions import Required, TypedDict
+
+import litellm
+from litellm._uuid import uuid
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.rerank import (
+ RerankBilledUnits,
+ RerankResponse,
+ RerankResponseMeta,
+ RerankResponseResult,
+)
+
+
+class NvidiaNimQueryObject(TypedDict):
+ text: Required[str]
+
+
+class NvidiaNimPassageObject(TypedDict):
+ text: Required[str]
+
+
+class NvidiaNimRerankRequest(TypedDict, total=False):
+ model: Required[str]
+ query: Required[NvidiaNimQueryObject]
+ passages: Required[List[NvidiaNimPassageObject]]
+ truncate: Literal["NONE", "END"]
+ top_k: int
+
+
+class NvidiaNimRankingResult(TypedDict):
+ index: Required[int]
+ logit: Required[float]
+
+
+class NvidiaNimRerankResponse(TypedDict):
+ rankings: Required[List[NvidiaNimRankingResult]]
+
+
+class NvidiaNimRerankConfig(BaseRerankConfig):
+ """
+ Reference: https://docs.api.nvidia.com/nim/reference/nvidia-llama-3_2-nv-rerankqa-1b-v2-infer
+
+ Nvidia NIM rerank API uses a different format:
+ - query is an object with 'text' field
+ - documents are called 'passages' and have 'text' field
+ """
+
+ DEFAULT_NIM_RERANK_API_BASE = "https://ai.api.nvidia.com"
+
+ def __init__(self) -> None:
+ pass
+
+ def _get_clean_model_name(self, model: str) -> str:
+ """Strip 'nvidia_nim/' prefix from model name if present."""
+ if model.startswith("nvidia_nim/"):
+ return model[len("nvidia_nim/") :]
+ return model
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ model: str,
+ optional_params: Optional[dict] = None,
+ ) -> str:
+ """
+ Construct the Nvidia NIM rerank URL.
+
+ Format: {api_base}/v1/retrieval/{model}/reranking
+
+ If the user provides a full URL (e.g., {api_base}/v1/retrieval/{model}/reranking),
+ it will be used as-is.
+ """
+ if not api_base:
+ api_base = self.DEFAULT_NIM_RERANK_API_BASE
+
+ api_base = api_base.rstrip("/")
+
+ # Check if user already provided the full URL with /retrieval/ path
+ if "/retrieval/" in api_base:
+ return api_base
+
+ # Ensure we don't have duplicate /v1
+ if api_base.endswith("/v1"):
+ api_base = api_base[:-3]
+
+ # Strip nvidia_nim/ prefix from model name if present
+ clean_model = self._get_clean_model_name(model)
+
+ return f"{api_base}/v1/retrieval/{clean_model}/reranking"
+
+ def get_supported_cohere_rerank_params(self, model: str) -> list:
+ """
+ Nvidia NIM supports these rerank parameters.
+ """
+ return [
+ "query",
+ "documents",
+ "top_n",
+ ]
+
+ def map_cohere_rerank_params(
+ self,
+ non_default_params: Optional[dict],
+ model: str,
+ drop_params: bool,
+ query: str,
+ documents: List[Union[str, Dict[str, Any]]],
+ custom_llm_provider: Optional[str] = None,
+ top_n: Optional[int] = None,
+ rank_fields: Optional[List[str]] = None,
+ return_documents: Optional[bool] = True,
+ max_chunks_per_doc: Optional[int] = None,
+ max_tokens_per_doc: Optional[int] = None,
+ ) -> Dict:
+ """
+ Map Cohere/OpenAI rerank params to Nvidia NIM format.
+
+ Parameter mapping:
+ - top_n (Cohere) -> top_k (Nvidia)
+
+ Nvidia NIM specific params (passed through as-is from non_default_params):
+ - truncate: How to truncate input if too long (NONE, END)
+ """
+ optional_nvidia_nim_rerank_params: Dict[str, Any] = {
+ "query": query,
+ "documents": documents,
+ }
+
+ # Map Cohere's top_n to Nvidia's top_k
+ if top_n is not None:
+ optional_nvidia_nim_rerank_params["top_k"] = top_n
+
+ # Pass through Nvidia-specific params from non_default_params
+ if non_default_params:
+ optional_nvidia_nim_rerank_params.update(non_default_params)
+ return dict(optional_nvidia_nim_rerank_params)
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ api_key: Optional[str] = None,
+ optional_params: Optional[dict] = None,
+ ) -> dict:
+ """
+ Validate that the Nvidia NIM API key is present.
+ """
+ if api_key is None:
+ api_key = get_secret_str("NVIDIA_NIM_API_KEY") or litellm.api_key
+
+ if api_key is None:
+ raise ValueError(
+ "Nvidia NIM API key is required. Please set 'NVIDIA_NIM_API_KEY' in your environment"
+ )
+
+ default_headers = {
+ "Authorization": f"Bearer {api_key}",
+ "accept": "application/json",
+ "content-type": "application/json",
+ }
+
+ # If 'Authorization' is provided in headers, it overrides the default
+ if "Authorization" in headers:
+ default_headers["Authorization"] = headers["Authorization"]
+
+ # Merge other headers, overriding any default ones except Authorization
+ return {**default_headers, **headers}
+
+ def transform_rerank_request(
+ self,
+ model: str,
+ optional_rerank_params: Dict,
+ headers: dict,
+ ) -> dict:
+ """
+ Transform request to Nvidia NIM format.
+
+ Nvidia NIM expects:
+ - query as {text: "..."}
+ - documents as passages: [{text: "..."}, ...]
+ - Optional: truncate (NONE or END), top_k
+
+ Note: optional_rerank_params may contain provider-specific params like 'top_k' and 'truncate'
+ that aren't in the OptionalRerankParams TypedDict but are passed through at runtime.
+ The mapping from Cohere's 'top_n' to Nvidia's 'top_k' already happened in map_cohere_rerank_params.
+ """
+ if "query" not in optional_rerank_params:
+ raise ValueError("query is required for Nvidia NIM rerank")
+ if "documents" not in optional_rerank_params:
+ raise ValueError("documents is required for Nvidia NIM rerank")
+
+ query = optional_rerank_params["query"]
+ documents = optional_rerank_params["documents"]
+
+ # Transform query to object format
+ query_obj: NvidiaNimQueryObject = {"text": query}
+
+ # Transform documents to passages format
+ passages: List[NvidiaNimPassageObject] = []
+ for doc in documents:
+ if isinstance(doc, str):
+ passages.append({"text": doc})
+ elif isinstance(doc, dict):
+ # If document is already a dict, check if it has 'text' field
+ if "text" in doc:
+ passages.append({"text": doc["text"]})
+ else:
+ # Otherwise, stringify the dict
+ import json
+
+ passages.append({"text": json.dumps(doc)})
+ else:
+ passages.append({"text": str(doc)})
+
+ # Strip nvidia_nim/ prefix from model name if present
+ clean_model = self._get_clean_model_name(model)
+
+ # Note: URL path uses underscores (llama-3_2) but JSON body uses periods (llama-3.2)
+ # Convert underscores back to periods for the model field in request body
+ model_for_body = clean_model.replace("_", ".")
+
+ # Build request using TypedDict
+ request_data: NvidiaNimRerankRequest = {
+ "model": model_for_body,
+ "query": query_obj,
+ "passages": passages,
+ }
+
+ # Add optional top_k parameter if provided (already mapped from top_n in map_cohere_rerank_params)
+ if "top_k" in optional_rerank_params and optional_rerank_params.get("top_k") is not None: # type: ignore
+ request_data["top_k"] = optional_rerank_params.get("top_k") # type: ignore
+
+ # Add Nvidia-specific truncate parameter if provided
+ # This is passed through from non_default_params, not in base OptionalRerankParams
+ if "truncate" in optional_rerank_params and optional_rerank_params.get("truncate") is not None: # type: ignore
+ truncate_value = optional_rerank_params.get("truncate") # type: ignore
+ if truncate_value in ["NONE", "END"]:
+ request_data["truncate"] = truncate_value # type: ignore
+
+ return dict(request_data)
+
+ def transform_rerank_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ model_response: RerankResponse,
+ logging_obj: LiteLLMLoggingObj,
+ api_key: Optional[str] = None,
+ request_data: dict = {},
+ optional_params: dict = {},
+ litellm_params: dict = {},
+ ) -> RerankResponse:
+ """
+ Transform Nvidia NIM rerank response to LiteLLM format.
+
+ Nvidia NIM returns (NvidiaNimRerankResponse):
+ {
+ "rankings": [
+ {
+ "index": 0,
+ "logit": 0.123
+ }
+ ]
+ }
+
+ LiteLLM expects (RerankResponse):
+ {
+ "results": [
+ {
+ "index": 0,
+ "relevance_score": 0.123,
+ "document": {"text": "..."} # optional
+ }
+ ]
+ }
+ """
+ try:
+ raw_response_json = raw_response.json()
+ except Exception:
+ raise BaseLLMException(
+ status_code=raw_response.status_code,
+ message=raw_response.text,
+ headers=raw_response.headers,
+ )
+
+ # Parse as NvidiaNimRerankResponse
+ nvidia_response: NvidiaNimRerankResponse = raw_response_json
+
+ # Transform Nvidia NIM response to LiteLLM format
+ results: List[RerankResponseResult] = []
+ rankings = nvidia_response.get("rankings", [])
+
+ # Get original documents from request if we need to include them
+ original_passages: List[NvidiaNimPassageObject] = request_data.get(
+ "passages", []
+ )
+
+ for ranking in rankings:
+ result_item: RerankResponseResult = {
+ "index": ranking["index"],
+ "relevance_score": ranking["logit"],
+ }
+
+ # Include document if it was in the original request
+ index: int = ranking["index"]
+ if index < len(original_passages):
+ result_item["document"] = {"text": original_passages[index]["text"]} # type: ignore
+
+ results.append(result_item)
+
+ # Construct metadata with billed_units
+ # Nvidia NIM uses "usage" field with "total_tokens"
+ usage = raw_response_json.get("usage", {})
+ total_tokens = usage.get("total_tokens", 0)
+
+ billed_units: RerankBilledUnits = {
+ "total_tokens": total_tokens if total_tokens > 0 else len(results)
+ }
+
+ meta: RerankResponseMeta = {"billed_units": billed_units}
+
+ return RerankResponse(
+ id=raw_response_json.get("id") or str(uuid.uuid4()),
+ results=results,
+ meta=meta,
+ )
+
+ def get_error_class(
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+ ) -> BaseLLMException:
+ return BaseLLMException(
+ status_code=status_code,
+ message=error_message,
+ headers=headers,
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oci/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oci/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..b1af7ed2ec3417cdd0b8aff794ea67019feaa579
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oci/chat/transformation.py
@@ -0,0 +1,1507 @@
+import base64
+import datetime
+import hashlib
+import json
+from dataclasses import dataclass
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ AsyncIterator,
+ Dict,
+ List,
+ Optional,
+ Protocol,
+ Tuple,
+ Union,
+)
+from urllib.parse import urlparse
+
+import httpx
+
+import litellm
+from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
+from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
+from litellm.llms.custom_httpx.http_handler import (
+ AsyncHTTPHandler,
+ HTTPHandler,
+ _get_httpx_client,
+ get_async_httpx_client,
+ version,
+)
+from litellm.llms.oci.common_utils import OCIError
+from litellm.types.llms.oci import (
+ CohereChatRequest,
+ CohereMessage,
+ CohereChatResult,
+ CohereParameterDefinition,
+ CohereStreamChunk,
+ CohereTool,
+ CohereToolCall,
+ OCIChatRequestPayload,
+ OCICompletionPayload,
+ OCICompletionResponse,
+ OCIContentPartUnion,
+ OCIImageContentPart,
+ OCIImageUrl,
+ OCIMessage,
+ OCIRoles,
+ OCIServingMode,
+ OCIStreamChunk,
+ OCITextContentPart,
+ OCIToolCall,
+ OCIToolDefinition,
+ OCIVendors,
+)
+from litellm.types.llms.openai import AllMessageValues
+from litellm.types.utils import (
+ Delta,
+ LlmProviders,
+ ModelResponse,
+ ModelResponseStream,
+ StreamingChoices,
+)
+from litellm.utils import (
+ ChatCompletionMessageToolCall,
+ CustomStreamWrapper,
+ Usage,
+)
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+ LiteLLMLoggingObj = Any
+
+
+class OCISignerProtocol(Protocol):
+ """
+ Protocol for OCI request signers (e.g., oci.signer.Signer).
+
+ This protocol defines the interface expected for OCI SDK signer objects.
+ Compatible with the OCI Python SDK's Signer class.
+
+ See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html
+ """
+
+ def do_request_sign(
+ self, request: Any, *, enforce_content_headers: bool = False
+ ) -> None:
+ """
+ Sign an HTTP request by adding authentication headers.
+
+ Args:
+ request: Request object with method, url, headers, body, and path_url attributes
+ enforce_content_headers: Whether to enforce content-type and content-length headers
+ """
+ ...
+
+
+@dataclass
+class OCIRequestWrapper:
+ """
+ Wrapper for HTTP requests compatible with OCI signer interface.
+
+ This class wraps request data in a format compatible with OCI SDK signers,
+ which expect objects with method, url, headers, body, and path_url attributes.
+ """
+
+ method: str
+ url: str
+ headers: dict
+ body: bytes
+
+ @property
+ def path_url(self) -> str:
+ """Returns the path + query string for OCI signing."""
+ parsed_url = urlparse(self.url)
+ return parsed_url.path + ("?" + parsed_url.query if parsed_url.query else "")
+
+
+def sha256_base64(data: bytes) -> str:
+ digest = hashlib.sha256(data).digest()
+ return base64.b64encode(digest).decode()
+
+
+def build_signature_string(method, path, headers, signed_headers):
+ lines = []
+ for header in signed_headers:
+ if header == "(request-target)":
+ value = f"{method.lower()} {path}"
+ else:
+ value = headers[header]
+ lines.append(f"{header}: {value}")
+ return "\n".join(lines)
+
+
+def load_private_key_from_str(key_str: str):
+ try:
+ from cryptography.hazmat.primitives import serialization
+ from cryptography.hazmat.primitives.asymmetric import rsa
+ except ImportError as e:
+ raise ImportError(
+ "cryptography package is required for OCI authentication. "
+ "Please install it with: pip install cryptography"
+ ) from e
+
+ key = serialization.load_pem_private_key(
+ key_str.encode("utf-8"),
+ password=None,
+ )
+ if not isinstance(key, rsa.RSAPrivateKey):
+ raise TypeError(
+ "The provided private key is not an RSA key, which is required for OCI signing."
+ )
+ return key
+
+
+def load_private_key_from_file(file_path: str):
+ """Loads a private key from a file path"""
+ try:
+ with open(file_path, "r", encoding="utf-8") as f:
+ key_str = f.read().strip()
+ except FileNotFoundError:
+ raise FileNotFoundError(f"Private key file not found: {file_path}")
+ except OSError as e:
+ raise OSError(f"Failed to read private key file '{file_path}': {e}") from e
+
+ if not key_str:
+ raise ValueError(f"Private key file is empty: {file_path}")
+
+ return load_private_key_from_str(key_str)
+
+
+def get_vendor_from_model(model: str) -> OCIVendors:
+ """
+ Extracts the vendor from the model name.
+ Args:
+ model (str): The model name.
+ Returns:
+ str: The vendor name.
+ """
+ vendor = model.split(".")[0].lower()
+ if vendor == "cohere":
+ return OCIVendors.COHERE
+ else:
+ return OCIVendors.GENERIC
+
+
+# 5 minute timeout (models may need to load)
+STREAMING_TIMEOUT = 60 * 5
+
+
+class OCIChatConfig(BaseConfig):
+ """
+ Configuration class for OCI's API interface.
+ """
+
+ def __init__(
+ self,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+ # mark the class as using a custom stream wrapper because the default only iterates on lines
+ setattr(self.__class__, "has_custom_stream_wrapper", True)
+
+ self.openai_to_oci_generic_param_map = {
+ "stream": "isStream",
+ "max_tokens": "maxTokens",
+ "max_completion_tokens": "maxTokens",
+ "temperature": "temperature",
+ "tools": "tools",
+ "frequency_penalty": "frequencyPenalty",
+ "logprobs": "logProbs",
+ "logit_bias": "logitBias",
+ "n": "numGenerations",
+ "presence_penalty": "presencePenalty",
+ "seed": "seed",
+ "stop": "stop",
+ "tool_choice": "toolChoice",
+ "top_p": "topP",
+ "max_retries": False,
+ "top_logprobs": False,
+ "modalities": False,
+ "prediction": False,
+ "stream_options": False,
+ "function_call": False,
+ "functions": False,
+ "extra_headers": False,
+ "parallel_tool_calls": False,
+ "audio": False,
+ "web_search_options": False,
+ "response_format": "responseFormat",
+ }
+
+ # Cohere and Gemini use the same parameter mapping as GENERIC
+ self.openai_to_oci_cohere_param_map = (
+ self.openai_to_oci_generic_param_map.copy()
+ )
+
+ def get_supported_openai_params(self, model: str) -> List[str]:
+ supported_params = []
+ vendor = get_vendor_from_model(model)
+ if vendor == OCIVendors.COHERE:
+ open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map
+ open_ai_to_oci_param_map.pop("tool_choice")
+ open_ai_to_oci_param_map.pop("max_retries")
+ else:
+ open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map
+ for key, value in open_ai_to_oci_param_map.items():
+ if value:
+ supported_params.append(key)
+
+ return supported_params
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ adapted_params = {}
+ vendor = get_vendor_from_model(model)
+ if vendor == OCIVendors.COHERE:
+ open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map
+ else:
+ open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map
+
+ all_params = {**non_default_params, **optional_params}
+
+ for key, value in all_params.items():
+ alias = open_ai_to_oci_param_map.get(key)
+
+ if alias is False:
+ # Workaround for mypy issue
+ if drop_params or litellm.drop_params:
+ continue
+ raise Exception(f"param `{key}` is not supported on OCI")
+
+ if alias is None:
+ adapted_params[key] = value
+ continue
+
+ adapted_params[alias] = value
+
+ if alias == "responseFormat":
+ adapted_params["response_format"] = value
+
+ return adapted_params
+
+ def _sign_with_oci_signer(
+ self,
+ headers: dict,
+ optional_params: dict,
+ request_data: dict,
+ api_base: str,
+ ) -> Tuple[dict, bytes]:
+ """
+ Sign request using OCI SDK Signer object.
+
+ Args:
+ headers: Request headers to be signed
+ optional_params: Optional parameters including oci_signer
+ request_data: The request body dict to be sent in HTTP request
+ api_base: The complete URL for the HTTP request
+
+ Returns:
+ Tuple of (signed_headers, encoded_body)
+
+ Raises:
+ OCIError: If signing fails
+ ValueError: If HTTP method is unsupported
+ """
+ oci_signer = optional_params.get("oci_signer")
+ body = json.dumps(request_data).encode("utf-8")
+ method = str(optional_params.get("method", "POST")).upper()
+
+ if method not in ["POST", "GET", "PUT", "DELETE", "PATCH"]:
+ raise ValueError(f"Unsupported HTTP method: {method}")
+
+ prepared_headers = headers.copy()
+ prepared_headers.setdefault("content-type", "application/json")
+ prepared_headers.setdefault("content-length", str(len(body)))
+
+ request_wrapper = OCIRequestWrapper(
+ method=method, url=api_base, headers=prepared_headers, body=body
+ )
+
+ if oci_signer is None:
+ raise ValueError(
+ "oci_signer cannot be None when calling _sign_with_oci_signer"
+ )
+
+ try:
+ oci_signer.do_request_sign(request_wrapper, enforce_content_headers=True)
+ except Exception as e:
+ raise OCIError(
+ status_code=500,
+ message=(
+ f"Failed to sign request with provided oci_signer: {str(e)}. "
+ "The signer must implement the OCI SDK Signer interface with a "
+ "do_request_sign(request, enforce_content_headers=True) method. "
+ "See: https://docs.oracle.com/en-us/iaas/tools/python/latest/api/signing.html"
+ ),
+ ) from e
+
+ headers.update(request_wrapper.headers)
+ return headers, body
+
+ def _sign_with_manual_credentials(
+ self,
+ headers: dict,
+ optional_params: dict,
+ request_data: dict,
+ api_base: str,
+ ) -> Tuple[dict, None]:
+ """
+ Sign request using manual OCI credentials.
+
+ Args:
+ headers: Request headers to be signed
+ optional_params: Optional parameters including OCI credentials
+ request_data: The request body dict to be sent in HTTP request
+ api_base: The complete URL for the HTTP request
+
+ Returns:
+ Tuple of (signed_headers, None)
+
+ Raises:
+ Exception: If required credentials are missing
+ ImportError: If cryptography package is not installed
+ """
+ oci_region = optional_params.get("oci_region", "us-ashburn-1")
+ api_base = (
+ api_base
+ or litellm.api_base
+ or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com"
+ )
+ oci_user = optional_params.get("oci_user")
+ oci_fingerprint = optional_params.get("oci_fingerprint")
+ oci_tenancy = optional_params.get("oci_tenancy")
+ oci_key = optional_params.get("oci_key")
+ oci_key_file = optional_params.get("oci_key_file")
+
+ if (
+ not oci_user
+ or not oci_fingerprint
+ or not oci_tenancy
+ or not (oci_key or oci_key_file)
+ ):
+ raise Exception(
+ "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, "
+ "and at least one of oci_key or oci_key_file."
+ )
+
+ method = str(optional_params.get("method", "POST")).upper()
+ body = json.dumps(request_data).encode("utf-8")
+ parsed = urlparse(api_base)
+ path = parsed.path or "/"
+ host = parsed.netloc
+
+ date = datetime.datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S GMT")
+ content_type = headers.get("content-type", "application/json")
+ content_length = str(len(body))
+ x_content_sha256 = sha256_base64(body)
+
+ headers_to_sign = {
+ "date": date,
+ "host": host,
+ "content-type": content_type,
+ "content-length": content_length,
+ "x-content-sha256": x_content_sha256,
+ }
+
+ signed_headers = [
+ "date",
+ "(request-target)",
+ "host",
+ "content-length",
+ "content-type",
+ "x-content-sha256",
+ ]
+ signing_string = build_signature_string(
+ method, path, headers_to_sign, signed_headers
+ )
+
+ try:
+ from cryptography.hazmat.primitives import hashes
+ from cryptography.hazmat.primitives.asymmetric import padding
+ except ImportError as e:
+ raise ImportError(
+ "cryptography package is required for OCI authentication. "
+ "Please install it with: pip install cryptography"
+ ) from e
+
+ # Handle oci_key - it should be a string (PEM content)
+ oci_key_content = None
+ if oci_key:
+ if isinstance(oci_key, str):
+ oci_key_content = oci_key
+ # Fix common issues with PEM content
+ # Replace escaped newlines with actual newlines
+ oci_key_content = oci_key_content.replace("\\n", "\n")
+ # Ensure proper line endings
+ if "\r\n" in oci_key_content:
+ oci_key_content = oci_key_content.replace("\r\n", "\n")
+ else:
+ raise OCIError(
+ status_code=400,
+ message=f"oci_key must be a string containing the PEM private key content. "
+ f"Got type: {type(oci_key).__name__}",
+ )
+
+ private_key = (
+ load_private_key_from_str(oci_key_content)
+ if oci_key_content
+ else load_private_key_from_file(oci_key_file)
+ if oci_key_file
+ else None
+ )
+
+ if private_key is None:
+ raise OCIError(
+ status_code=400,
+ message="Private key is required for OCI authentication. Please provide either oci_key or oci_key_file.",
+ )
+
+ signature = private_key.sign(
+ signing_string.encode("utf-8"),
+ padding.PKCS1v15(),
+ hashes.SHA256(),
+ )
+ signature_b64 = base64.b64encode(signature).decode()
+
+ key_id = f"{oci_tenancy}/{oci_user}/{oci_fingerprint}"
+
+ authorization = (
+ 'Signature version="1",'
+ f'keyId="{key_id}",'
+ 'algorithm="rsa-sha256",'
+ f'headers="{" ".join(signed_headers)}",'
+ f'signature="{signature_b64}"'
+ )
+
+ headers.update(
+ {
+ "authorization": authorization,
+ "date": date,
+ "host": host,
+ "content-type": content_type,
+ "content-length": content_length,
+ "x-content-sha256": x_content_sha256,
+ }
+ )
+
+ return headers, None
+
+ def sign_request(
+ self,
+ headers: dict,
+ optional_params: dict,
+ request_data: dict,
+ api_base: str,
+ api_key: Optional[str] = None,
+ model: Optional[str] = None,
+ stream: Optional[bool] = None,
+ fake_stream: Optional[bool] = None,
+ ) -> Tuple[dict, Optional[bytes]]:
+ """
+ Sign the OCI request by adding authentication headers.
+
+ Supports two signing modes:
+ 1. OCI SDK Signer: Use an oci_signer object to sign the request
+ 2. Manual Signing: Use OCI credentials to manually sign the request
+
+ Args:
+ headers: Request headers to be signed
+ optional_params: Optional parameters including auth credentials or oci_signer
+ request_data: The request body dict to be sent in HTTP request
+ api_base: The complete URL for the HTTP request
+ api_key: Optional API key (not used for OCI)
+ model: Optional model name
+ stream: Optional streaming flag
+ fake_stream: Optional fake streaming flag
+
+ Returns:
+ Tuple of (signed_headers, encoded_body):
+ - If oci_signer is provided: Returns (headers, body) where body is the encoded JSON
+ - If manual credentials are provided: Returns (headers, None) as body is not returned
+ for the manual signing path
+
+ Raises:
+ OCIError: If signing fails with oci_signer
+ Exception: If required credentials are missing
+ ImportError: If cryptography package is not installed (manual signing only)
+
+ Example:
+ >>> from oci.signer import Signer
+ >>> signer = Signer(
+ ... tenancy="ocid1.tenancy.oc1..",
+ ... user="ocid1.user.oc1..",
+ ... fingerprint="xx:xx:xx",
+ ... private_key_file_location="~/.oci/key.pem"
+ ... )
+ >>> headers, body = config.sign_request(
+ ... headers={},
+ ... optional_params={"oci_signer": signer},
+ ... request_data={"message": "Hello"},
+ ... api_base="https://inference.generativeai.us-ashburn-1.oci.oraclecloud.com/..."
+ ... )
+ """
+ oci_signer = optional_params.get("oci_signer")
+
+ # If a signer is provided, use it for request signing
+ if oci_signer is not None:
+ return self._sign_with_oci_signer(
+ headers, optional_params, request_data, api_base
+ )
+
+ # Standard manual credential signing
+ return self._sign_with_manual_credentials(
+ headers, optional_params, request_data, api_base
+ )
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ """
+ Validate the OCI environment and credentials.
+
+ Supports two authentication modes:
+ 1. OCI SDK Signer: Pass an oci_signer object (e.g., oci.signer.Signer)
+ 2. Manual Credentials: Pass oci_user, oci_fingerprint, oci_tenancy, and oci_key/oci_key_file
+
+ Args:
+ headers: Request headers to populate
+ model: Model name
+ messages: List of chat messages
+ optional_params: Optional parameters including authentication credentials
+ litellm_params: LiteLLM parameters
+ api_key: Optional API key (not used for OCI)
+ api_base: Optional API base URL
+
+ Returns:
+ Updated headers dict
+
+ Raises:
+ Exception: If required parameters are missing or invalid
+ """
+ oci_signer = optional_params.get("oci_signer")
+ oci_region = optional_params.get("oci_region", "us-ashburn-1")
+
+ # Determine api_base
+ api_base = (
+ api_base
+ or litellm.api_base
+ or f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com"
+ )
+
+ if not api_base:
+ raise Exception(
+ "Either `api_base` must be provided or `litellm.api_base` must be set. "
+ "Alternatively, you can set the `oci_region` optional parameter to use the default OCI region."
+ )
+
+ # Validate credentials only if signer is not provided
+ if oci_signer is None:
+ oci_user = optional_params.get("oci_user")
+ oci_fingerprint = optional_params.get("oci_fingerprint")
+ oci_tenancy = optional_params.get("oci_tenancy")
+ oci_key = optional_params.get("oci_key")
+ oci_key_file = optional_params.get("oci_key_file")
+ oci_compartment_id = optional_params.get("oci_compartment_id")
+
+ if (
+ not oci_user
+ or not oci_fingerprint
+ or not oci_tenancy
+ or not (oci_key or oci_key_file)
+ or not oci_compartment_id
+ ):
+ raise Exception(
+ "Missing required parameters: oci_user, oci_fingerprint, oci_tenancy, oci_compartment_id "
+ "and at least one of oci_key or oci_key_file. "
+ "Alternatively, provide an oci_signer object from the OCI SDK."
+ )
+
+ # Common header setup
+ headers.update(
+ {
+ "content-type": "application/json",
+ "user-agent": f"litellm/{version}",
+ }
+ )
+
+ if not messages:
+ raise Exception(
+ "kwarg `messages` must be an array of messages that follow the openai chat standard"
+ )
+
+ return headers
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ api_key: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: dict,
+ stream: Optional[bool] = None,
+ ) -> str:
+ oci_region = optional_params.get("oci_region", "us-ashburn-1")
+ return f"https://inference.generativeai.{oci_region}.oci.oraclecloud.com/20231130/actions/chat"
+
+ def _get_optional_params(self, vendor: OCIVendors, optional_params: dict) -> Dict:
+ selected_params = {}
+ if vendor == OCIVendors.COHERE:
+ open_ai_to_oci_param_map = self.openai_to_oci_cohere_param_map
+ # remove tool_choice from the map
+ open_ai_to_oci_param_map.pop("tool_choice")
+ # Add default values for Cohere API
+ selected_params = {
+ "maxTokens": 600,
+ "temperature": 1,
+ "topK": 0,
+ "topP": 0.75,
+ "frequencyPenalty": 0,
+ }
+ else:
+ open_ai_to_oci_param_map = self.openai_to_oci_generic_param_map
+
+ # Map OpenAI params to OCI params
+ for openai_key, oci_key in open_ai_to_oci_param_map.items():
+ if oci_key and openai_key in optional_params:
+ selected_params[oci_key] = optional_params[openai_key] # type: ignore[index]
+
+ # Also check for already-mapped OCI params (for backward compatibility)
+ for oci_value in open_ai_to_oci_param_map.values():
+ if (
+ oci_value
+ and oci_value in optional_params
+ and oci_value not in selected_params
+ ):
+ selected_params[oci_value] = optional_params[oci_value] # type: ignore[index]
+
+ if "tools" in selected_params:
+ if vendor == OCIVendors.COHERE:
+ selected_params["tools"] = self.adapt_tool_definitions_to_cohere_standard( # type: ignore[assignment]
+ selected_params["tools"] # type: ignore[arg-type]
+ )
+ else:
+ selected_params["tools"] = adapt_tool_definition_to_oci_standard( # type: ignore[assignment]
+ selected_params["tools"], vendor # type: ignore[arg-type]
+ )
+
+ # Transform response_format type to OCI uppercase format
+ if "responseFormat" in selected_params:
+ rf = selected_params["responseFormat"]
+ if isinstance(rf, dict) and "type" in rf:
+ rf_payload = dict(rf)
+ selected_params["responseFormat"] = rf_payload
+
+ response_type = rf_payload["type"]
+ schema_payload: Optional[Any] = None
+
+ if "json_schema" in rf_payload:
+ raw_schema_payload = rf_payload.pop("json_schema")
+ if isinstance(raw_schema_payload, dict):
+ schema_payload = dict(raw_schema_payload)
+ else:
+ schema_payload = raw_schema_payload
+
+ if schema_payload is not None:
+ rf_payload["jsonSchema"] = schema_payload
+
+ if vendor == OCIVendors.COHERE:
+ # Cohere expects lower-case type values
+ rf_payload["type"] = response_type
+ else:
+ format_type = response_type.upper()
+ if format_type == "JSON":
+ format_type = "JSON_OBJECT"
+ rf_payload["type"] = format_type
+
+ return selected_params
+
+ def adapt_messages_to_cohere_standard(
+ self, messages: List[AllMessageValues]
+ ) -> List[CohereMessage]:
+ """Build chat history for Cohere models."""
+ chat_history = []
+ for msg in messages[:-1]: # All messages except the last one
+ role = msg.get("role")
+ content = msg.get("content")
+
+ if isinstance(content, list):
+ # Extract text from content array
+ text_content = ""
+ for content_item in content:
+ if (
+ isinstance(content_item, dict)
+ and content_item.get("type") == "text"
+ ):
+ text_content += content_item.get("text", "")
+ content = text_content
+
+ # Ensure content is a string
+ if not isinstance(content, str):
+ content = str(content) if content is not None else ""
+
+ # Handle tool calls
+ tool_calls: Optional[List[CohereToolCall]] = None
+ if role == "assistant" and "tool_calls" in msg and msg.get("tool_calls"): # type: ignore[union-attr,typeddict-item]
+ tool_calls = []
+ for tool_call in msg["tool_calls"]: # type: ignore[union-attr,typeddict-item]
+ # Parse arguments if they're a JSON string
+ raw_arguments: Any = tool_call.get("function", {}).get(
+ "arguments", {}
+ )
+ if isinstance(raw_arguments, str):
+ try:
+ arguments: Dict[str, Any] = json.loads(raw_arguments)
+ except json.JSONDecodeError:
+ arguments = {}
+ else:
+ arguments = raw_arguments
+
+ tool_calls.append(
+ CohereToolCall(
+ name=str(tool_call.get("function", {}).get("name", "")),
+ parameters=arguments,
+ )
+ )
+
+ if role == "user":
+ chat_history.append(CohereMessage(role="USER", message=content))
+ elif role == "assistant":
+ chat_history.append(
+ CohereMessage(role="CHATBOT", message=content, toolCalls=tool_calls)
+ )
+ elif role == "tool":
+ # Tool messages need special handling
+ chat_history.append(
+ CohereMessage(
+ role="TOOL",
+ message=content,
+ toolCalls=None, # Tool messages don't have tool calls
+ )
+ )
+
+ return chat_history
+
+ def adapt_tool_definitions_to_cohere_standard(
+ self, tools: List[Dict[str, Any]]
+ ) -> List[CohereTool]:
+ """Adapt tool definitions to Cohere format."""
+ cohere_tools = []
+ for tool in tools:
+ function_def = tool.get("function", {})
+ parameters = function_def.get("parameters", {}).get("properties", {})
+ required = function_def.get("parameters", {}).get("required", [])
+
+ parameter_definitions = {}
+ for param_name, param_schema in parameters.items():
+ parameter_definitions[param_name] = CohereParameterDefinition(
+ description=param_schema.get("description", ""),
+ type=param_schema.get("type", "string"),
+ isRequired=param_name in required,
+ )
+
+ cohere_tools.append(
+ CohereTool(
+ name=function_def.get("name", ""),
+ description=function_def.get("description", ""),
+ parameterDefinitions=parameter_definitions,
+ )
+ )
+
+ return cohere_tools
+
+ def _extract_text_content(self, content: Any) -> str:
+ """Extract text content from message content."""
+ if isinstance(content, str):
+ return content
+ elif isinstance(content, list):
+ text_content = ""
+ for content_item in content:
+ if (
+ isinstance(content_item, dict)
+ and content_item.get("type") == "text"
+ ):
+ text_content += content_item.get("text", "")
+ return text_content
+ return str(content)
+
+ def transform_request(
+ self,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ headers: dict,
+ ) -> dict:
+ oci_compartment_id = optional_params.get("oci_compartment_id", None)
+ if not oci_compartment_id:
+ raise Exception("kwarg `oci_compartment_id` is required for OCI requests")
+
+ vendor = get_vendor_from_model(model)
+
+ oci_serving_mode = optional_params.get("oci_serving_mode", "ON_DEMAND")
+ if oci_serving_mode not in ["ON_DEMAND", "DEDICATED"]:
+ raise Exception(
+ "kwarg `oci_serving_mode` must be either 'ON_DEMAND' or 'DEDICATED'"
+ )
+
+ if oci_serving_mode == "DEDICATED":
+ oci_endpoint_id = optional_params.get("oci_endpoint_id", model)
+ servingMode = OCIServingMode(
+ servingType="DEDICATED",
+ endpointId=oci_endpoint_id,
+ )
+ else:
+ servingMode = OCIServingMode(
+ servingType="ON_DEMAND",
+ modelId=model,
+ )
+
+ # Build request based on vendor type
+ if vendor == OCIVendors.COHERE:
+ # For Cohere, we need to use the specific Cohere format
+ # Extract the last user message as the main message
+ user_messages = [msg for msg in messages if msg.get("role") == "user"]
+ if not user_messages:
+ raise Exception("No user message found for Cohere model")
+
+ # Extract system messages into preambleOverride
+ system_messages = [msg for msg in messages if msg.get("role") == "system"]
+ preamble_override = None
+ if system_messages:
+ preamble = "\n".join(
+ self._extract_text_content(msg["content"])
+ for msg in system_messages
+ )
+ if preamble:
+ preamble_override = preamble
+
+ # Create Cohere-specific chat request
+ optional_cohere_params = self._get_optional_params(
+ OCIVendors.COHERE, optional_params
+ )
+ chat_request = CohereChatRequest(
+ apiFormat="COHERE",
+ message=self._extract_text_content(user_messages[-1]["content"]),
+ chatHistory=self.adapt_messages_to_cohere_standard(messages),
+ preambleOverride=preamble_override,
+ **optional_cohere_params,
+ )
+
+ data = OCICompletionPayload(
+ compartmentId=oci_compartment_id,
+ servingMode=servingMode,
+ chatRequest=chat_request,
+ )
+ else:
+ # Use generic format for other vendors
+ data = OCICompletionPayload(
+ compartmentId=oci_compartment_id,
+ servingMode=servingMode,
+ chatRequest=OCIChatRequestPayload(
+ apiFormat=vendor.value,
+ messages=adapt_messages_to_generic_oci_standard(messages),
+ **self._get_optional_params(vendor, optional_params),
+ ),
+ )
+
+ return data.model_dump(exclude_none=True)
+
+ def _handle_cohere_response(
+ self, json_response: dict, model: str, model_response: ModelResponse
+ ) -> ModelResponse:
+ """Handle Cohere-specific response format."""
+ cohere_response = CohereChatResult(**json_response)
+ # Cohere response format (uses camelCase)
+ model_id = model
+
+ # Set basic response info
+ model_response.model = model_id
+ model_response.created = int(datetime.datetime.now().timestamp())
+
+ # Extract the response text
+ response_text = cohere_response.chatResponse.text
+ oci_finish_reason = cohere_response.chatResponse.finishReason
+
+ # Map finish reason
+ if oci_finish_reason == "COMPLETE":
+ finish_reason = "stop"
+ elif oci_finish_reason == "MAX_TOKENS":
+ finish_reason = "length"
+ else:
+ finish_reason = "stop"
+
+ # Handle tool calls
+ tool_calls: Optional[List[Dict[str, Any]]] = None
+ if cohere_response.chatResponse.toolCalls:
+ tool_calls = []
+ for tool_call in cohere_response.chatResponse.toolCalls:
+ tool_calls.append(
+ {
+ "id": f"call_{len(tool_calls)}", # Generate a simple ID
+ "type": "function",
+ "function": {
+ "name": tool_call.name,
+ "arguments": json.dumps(tool_call.parameters),
+ },
+ }
+ )
+
+ # Create choice
+ from litellm.types.utils import Choices
+
+ choice = Choices(
+ index=0,
+ message={
+ "role": "assistant",
+ "content": response_text,
+ "tool_calls": tool_calls,
+ },
+ finish_reason=finish_reason,
+ )
+ model_response.choices = [choice]
+
+ # Extract usage info
+ usage_info = cohere_response.chatResponse.usage
+ from litellm.types.utils import Usage
+
+ model_response.usage = Usage( # type: ignore[attr-defined]
+ prompt_tokens=usage_info.promptTokens, # type: ignore[union-attr]
+ completion_tokens=usage_info.completionTokens, # type: ignore[union-attr]
+ total_tokens=usage_info.totalTokens, # type: ignore[union-attr]
+ )
+
+ return model_response
+
+ def _handle_generic_response(
+ self,
+ json: dict,
+ model: str,
+ model_response: ModelResponse,
+ raw_response: httpx.Response,
+ ) -> ModelResponse:
+ """Handle generic OCI response format."""
+ try:
+ completion_response = OCICompletionResponse(**json)
+ except TypeError as e:
+ raise OCIError(
+ message=f"Response cannot be casted to OCICompletionResponse: {str(e)}",
+ status_code=raw_response.status_code,
+ )
+
+ iso_str = completion_response.chatResponse.timeCreated
+ dt = datetime.datetime.fromisoformat(iso_str.replace("Z", "+00:00"))
+ model_response.created = int(dt.timestamp())
+
+ model_response.model = completion_response.modelId
+
+ message = model_response.choices[0].message # type: ignore
+ response_message = completion_response.chatResponse.choices[0].message
+ if response_message.content and response_message.content[0].type == "TEXT":
+ message.content = response_message.content[0].text
+ if response_message.toolCalls:
+ message.tool_calls = adapt_tools_to_openai_standard(
+ response_message.toolCalls
+ )
+
+ usage = Usage(
+ prompt_tokens=completion_response.chatResponse.usage.promptTokens,
+ completion_tokens=completion_response.chatResponse.usage.completionTokens,
+ total_tokens=completion_response.chatResponse.usage.totalTokens,
+ )
+ model_response.usage = usage # type: ignore
+
+ return model_response
+
+ def transform_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ model_response: ModelResponse,
+ logging_obj: LiteLLMLoggingObj,
+ request_data: dict,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: Any,
+ api_key: Optional[str] = None,
+ json_mode: Optional[bool] = None,
+ ) -> ModelResponse:
+ json = raw_response.json() # noqa: F811
+
+ error = json.get("error")
+
+ if error is not None:
+ raise OCIError(
+ message=str(json["error"]),
+ status_code=raw_response.status_code,
+ )
+
+ if not isinstance(json, dict):
+ raise OCIError(
+ message="Invalid response format from OCI",
+ status_code=raw_response.status_code,
+ )
+
+ vendor = get_vendor_from_model(model)
+
+ # Handle response based on vendor type
+ if vendor == OCIVendors.COHERE:
+ model_response = self._handle_cohere_response(json, model, model_response)
+ else:
+ model_response = self._handle_generic_response(
+ json, model, model_response, raw_response
+ )
+
+ model_response._hidden_params["additional_headers"] = raw_response.headers
+
+ return model_response
+
+ @track_llm_api_timing()
+ def get_sync_custom_stream_wrapper(
+ self,
+ model: str,
+ custom_llm_provider: str,
+ logging_obj: LiteLLMLoggingObj,
+ api_base: str,
+ headers: dict,
+ data: dict,
+ messages: list,
+ client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
+ json_mode: Optional[bool] = None,
+ signed_json_body: Optional[bytes] = None,
+ ) -> "OCIStreamWrapper":
+ if "stream" in data:
+ del data["stream"]
+ if client is None or isinstance(client, AsyncHTTPHandler):
+ client = _get_httpx_client(params={})
+
+ try:
+ response = client.post(
+ api_base,
+ headers=headers,
+ data=json.dumps(data),
+ stream=True,
+ logging_obj=logging_obj,
+ timeout=STREAMING_TIMEOUT,
+ )
+ except httpx.HTTPStatusError as e:
+ raise OCIError(status_code=e.response.status_code, message=e.response.text)
+
+ if response.status_code != 200:
+ raise OCIError(status_code=response.status_code, message=response.text)
+
+ completion_stream = response.iter_text()
+
+ streaming_response = OCIStreamWrapper(
+ completion_stream=completion_stream,
+ model=model,
+ custom_llm_provider=custom_llm_provider,
+ logging_obj=logging_obj,
+ )
+ return streaming_response
+
+ @track_llm_api_timing()
+ async def get_async_custom_stream_wrapper(
+ self,
+ model: str,
+ custom_llm_provider: str,
+ logging_obj: LiteLLMLoggingObj,
+ api_base: str,
+ headers: dict,
+ data: dict,
+ messages: list,
+ client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
+ json_mode: Optional[bool] = None,
+ signed_json_body: Optional[bytes] = None,
+ ) -> "OCIStreamWrapper":
+ if "stream" in data:
+ del data["stream"]
+
+ if client is None or isinstance(client, HTTPHandler):
+ client = get_async_httpx_client(llm_provider=LlmProviders.BYTEZ, params={})
+
+ try:
+ response = await client.post(
+ api_base,
+ headers=headers,
+ data=json.dumps(data),
+ stream=True,
+ logging_obj=logging_obj,
+ timeout=STREAMING_TIMEOUT,
+ )
+ except httpx.HTTPStatusError as e:
+ raise OCIError(status_code=e.response.status_code, message=e.response.text)
+
+ if response.status_code != 200:
+ raise OCIError(status_code=response.status_code, message=response.text)
+
+ completion_stream = response.aiter_text()
+
+ async def split_chunks(completion_stream: AsyncIterator[str]):
+ async for item in completion_stream:
+ for chunk in item.split("\n\n"):
+ if not chunk:
+ continue
+ yield chunk.strip()
+
+ streaming_response = OCIStreamWrapper(
+ completion_stream=split_chunks(completion_stream),
+ model=model,
+ custom_llm_provider=custom_llm_provider,
+ logging_obj=logging_obj,
+ )
+ return streaming_response
+
+ def get_error_class(
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+ ) -> BaseLLMException:
+ return OCIError(status_code=status_code, message=error_message)
+
+
+open_ai_to_generic_oci_role_map: Dict[str, OCIRoles] = {
+ "system": "SYSTEM",
+ "user": "USER",
+ "assistant": "ASSISTANT",
+ "tool": "TOOL",
+}
+
+
+def adapt_messages_to_generic_oci_standard_content_message(
+ role: str, content: Union[str, list]
+) -> OCIMessage:
+ new_content: List[OCIContentPartUnion] = []
+ if isinstance(content, str):
+ return OCIMessage(
+ role=open_ai_to_generic_oci_role_map[role],
+ content=[OCITextContentPart(text=content)],
+ toolCalls=None,
+ toolCallId=None,
+ )
+
+ # content is a list of content items:
+ # [
+ # {"type": "text", "text": "Hello"},
+ # {"type": "image_url", "image_url": "https://example.com/image.png"}
+ # ]
+ for content_item in content:
+ if not isinstance(content_item, dict):
+ raise Exception("Each content item must be a dictionary")
+
+ type = content_item.get("type")
+ if not isinstance(type, str):
+ raise Exception("Prop `type` is not a string")
+
+ if type not in ["text", "image_url"]:
+ raise Exception(f"Prop `{type}` is not supported")
+
+ if type == "text":
+ text = content_item.get("text")
+ if not isinstance(text, str):
+ raise Exception("Prop `text` is not a string")
+ new_content.append(OCITextContentPart(text=text))
+
+ elif type == "image_url":
+ image_url = content_item.get("image_url")
+ # Handle both OpenAI format (object with url) and string format
+ if isinstance(image_url, dict):
+ image_url = image_url.get("url")
+ if not isinstance(image_url, str):
+ raise Exception(
+ "Prop `image_url` must be a string or an object with a `url` property"
+ )
+ new_content.append(OCIImageContentPart(imageUrl=OCIImageUrl(url=image_url)))
+
+ return OCIMessage(
+ role=open_ai_to_generic_oci_role_map[role],
+ content=new_content,
+ toolCalls=None,
+ toolCallId=None,
+ )
+
+
+def adapt_messages_to_generic_oci_standard_tool_call(
+ role: str, tool_calls: list
+) -> OCIMessage:
+ tool_calls_formated = []
+ for tool_call in tool_calls:
+ if not isinstance(tool_call, dict):
+ raise Exception("Each tool call must be a dictionary")
+
+ if tool_call.get("type") != "function":
+ raise Exception("OCI only supports function tools")
+
+ tool_call_id = tool_call.get("id")
+ if not isinstance(tool_call_id, str):
+ raise Exception("Prop `id` is not a string")
+
+ tool_function = tool_call.get("function")
+ if not isinstance(tool_function, dict):
+ raise Exception("Prop `function` is not a dictionary")
+
+ function_name = tool_function.get("name")
+ if not isinstance(function_name, str):
+ raise Exception("Prop `name` is not a string")
+
+ arguments = tool_call["function"].get("arguments", "{}")
+ if not isinstance(arguments, str):
+ raise Exception("Prop `arguments` is not a string")
+
+ # tool_calls_formated.append(OCIToolCall(
+ # id=tool_call_id,
+ # type="FUNCTION",
+ # function=OCIFunction(
+ # name=function_name,
+ # arguments=arguments
+ # )
+ # ))
+
+ tool_calls_formated.append(
+ OCIToolCall(
+ id=tool_call_id,
+ type="FUNCTION",
+ name=function_name,
+ arguments=arguments,
+ )
+ )
+
+ return OCIMessage(
+ role=open_ai_to_generic_oci_role_map[role],
+ content=None,
+ toolCalls=tool_calls_formated,
+ toolCallId=None,
+ )
+
+
+def adapt_messages_to_generic_oci_standard_tool_response(
+ role: str, tool_call_id: str, content: str
+) -> OCIMessage:
+ return OCIMessage(
+ role=open_ai_to_generic_oci_role_map[role],
+ content=[OCITextContentPart(text=content)],
+ toolCalls=None,
+ toolCallId=tool_call_id,
+ )
+
+
+def adapt_messages_to_generic_oci_standard(
+ messages: List[AllMessageValues],
+) -> List[OCIMessage]:
+ new_messages = []
+ for message in messages:
+ role = message["role"]
+ content = message.get("content")
+ tool_calls = message.get("tool_calls")
+ tool_call_id = message.get("tool_call_id")
+
+ if role == "assistant" and tool_calls is not None:
+ if not isinstance(tool_calls, list):
+ raise Exception("Prop `tool_calls` must be a list of tool calls")
+ new_messages.append(
+ adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls)
+ )
+
+ elif role in ["system", "user", "assistant"] and content is not None:
+ if not isinstance(content, (str, list)):
+ raise Exception(
+ "Prop `content` must be a string or a list of content items"
+ )
+ new_messages.append(
+ adapt_messages_to_generic_oci_standard_content_message(role, content)
+ )
+
+ elif role == "tool":
+ if not isinstance(tool_call_id, str):
+ raise Exception("Prop `tool_call_id` is required and must be a string")
+ if not isinstance(content, str):
+ raise Exception("Prop `content` is not a string")
+ new_messages.append(
+ adapt_messages_to_generic_oci_standard_tool_response(
+ role, tool_call_id, content
+ )
+ )
+
+ return new_messages
+
+
+def adapt_tool_definition_to_oci_standard(tools: List[Dict], vendor: OCIVendors):
+ new_tools = []
+ for tool in tools:
+ if tool["type"] != "function":
+ raise Exception("OCI only supports function tools")
+
+ tool_function = tool.get("function")
+ if not isinstance(tool_function, dict):
+ raise Exception("Prop `function` is not a dictionary")
+
+ new_tool = OCIToolDefinition(
+ type="FUNCTION",
+ name=tool_function.get("name"),
+ description=tool_function.get("description", ""),
+ parameters=tool_function.get("parameters", {}),
+ )
+ new_tools.append(new_tool)
+
+ return new_tools
+
+
+def adapt_tools_to_openai_standard(
+ tools: List[OCIToolCall],
+) -> List[ChatCompletionMessageToolCall]:
+ new_tools = []
+ for tool in tools:
+ new_tool = ChatCompletionMessageToolCall(
+ id=tool.id,
+ type="function",
+ function={
+ "name": tool.name,
+ "arguments": tool.arguments,
+ },
+ )
+ new_tools.append(new_tool)
+ return new_tools
+
+
+class OCIStreamWrapper(CustomStreamWrapper):
+ """
+ Custom stream wrapper for OCI responses.
+ This class is used to handle streaming responses from OCI's API.
+ """
+
+ def __init__(
+ self,
+ **kwargs: Any,
+ ):
+ super().__init__(**kwargs)
+
+ def chunk_creator(self, chunk: Any):
+ if not isinstance(chunk, str):
+ raise ValueError(f"Chunk is not a string: {chunk}")
+ if not chunk.startswith("data:"):
+ raise ValueError(f"Chunk does not start with 'data:': {chunk}")
+ dict_chunk = json.loads(chunk[5:]) # Remove 'data: ' prefix and parse JSON
+
+ # Check if this is a Cohere stream chunk
+ if "apiFormat" in dict_chunk and dict_chunk.get("apiFormat") == "COHERE":
+ return self._handle_cohere_stream_chunk(dict_chunk)
+ else:
+ return self._handle_generic_stream_chunk(dict_chunk)
+
+ def _handle_cohere_stream_chunk(self, dict_chunk: dict):
+ """Handle Cohere-specific streaming chunks."""
+ try:
+ typed_chunk = CohereStreamChunk(**dict_chunk)
+ except TypeError as e:
+ raise ValueError(f"Chunk cannot be casted to CohereStreamChunk: {str(e)}")
+
+ if typed_chunk.index is None:
+ typed_chunk.index = 0
+
+ # Extract text content
+ text = typed_chunk.text or ""
+
+ # Map finish reason to standard format
+ finish_reason = typed_chunk.finishReason
+ if finish_reason == "COMPLETE":
+ finish_reason = "stop"
+ elif finish_reason == "MAX_TOKENS":
+ finish_reason = "length"
+ elif finish_reason is None:
+ finish_reason = None
+ else:
+ finish_reason = "stop"
+
+ # For Cohere, we don't have tool calls in the streaming format
+ tool_calls = None
+
+ return ModelResponseStream(
+ choices=[
+ StreamingChoices(
+ index=typed_chunk.index if typed_chunk.index else 0,
+ delta=Delta(
+ content=text,
+ tool_calls=tool_calls,
+ provider_specific_fields=None,
+ thinking_blocks=None,
+ reasoning_content=None,
+ ),
+ finish_reason=finish_reason,
+ )
+ ]
+ )
+
+ def _handle_generic_stream_chunk(self, dict_chunk: dict):
+ """Handle generic OCI streaming chunks."""
+ # Fix missing required fields in tool calls before Pydantic validation
+ # OCI streams tool calls progressively, so early chunks may be missing required fields
+ if dict_chunk.get("message") and dict_chunk["message"].get("toolCalls"):
+ for tool_call in dict_chunk["message"]["toolCalls"]:
+ if "arguments" not in tool_call:
+ tool_call["arguments"] = ""
+ if "id" not in tool_call:
+ tool_call["id"] = ""
+ if "name" not in tool_call:
+ tool_call["name"] = ""
+
+ try:
+ typed_chunk = OCIStreamChunk(**dict_chunk)
+ except TypeError as e:
+ raise ValueError(f"Chunk cannot be casted to OCIStreamChunk: {str(e)}")
+
+ if typed_chunk.index is None:
+ typed_chunk.index = 0
+
+ text = ""
+ if typed_chunk.message and typed_chunk.message.content:
+ for item in typed_chunk.message.content:
+ if isinstance(item, OCITextContentPart):
+ text += item.text
+ elif isinstance(item, OCIImageContentPart):
+ raise ValueError(
+ "OCI does not support image content in streaming responses"
+ )
+ else:
+ raise ValueError(
+ f"Unsupported content type in OCI response: {item.type}"
+ )
+
+ tool_calls = None
+ if typed_chunk.message and typed_chunk.message.toolCalls:
+ tool_calls = adapt_tools_to_openai_standard(typed_chunk.message.toolCalls)
+
+ return ModelResponseStream(
+ choices=[
+ StreamingChoices(
+ index=typed_chunk.index if typed_chunk.index else 0,
+ delta=Delta(
+ content=text,
+ tool_calls=(
+ [tool.model_dump() for tool in tool_calls]
+ if tool_calls
+ else None
+ ),
+ provider_specific_fields=None, # OCI does not have provider specific fields in the response
+ thinking_blocks=None, # OCI does not have thinking blocks in the response
+ reasoning_content=None, # OCI does not have reasoning content in the response
+ ),
+ finish_reason=typed_chunk.finishReason,
+ )
+ ]
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oci/common_utils.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oci/common_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..661a6c89e4b8286b117f19d57a007dc0e4eef663
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oci/common_utils.py
@@ -0,0 +1,19 @@
+from typing import Optional
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+
+class OCIError(BaseLLMException):
+ def __init__(
+ self,
+ status_code: int,
+ message: str,
+ headers: Optional[httpx.Headers] = None,
+ ):
+ super().__init__(
+ status_code=status_code,
+ message=message,
+ headers=headers,
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..3d9618dfed00644f9855743d7f488ef103ce1c30
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/chat/transformation.py
@@ -0,0 +1,578 @@
+import json
+import time
+from litellm._uuid import uuid
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ AsyncIterator,
+ Iterator,
+ List,
+ Optional,
+ Union,
+ cast,
+)
+
+from httpx._models import Headers, Response
+from pydantic import BaseModel
+
+import litellm
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+ _extract_reasoning_content,
+ convert_content_list_to_str,
+ extract_images_from_message,
+)
+from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
+from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
+from litellm.types.llms.ollama import (
+ OllamaChatCompletionMessage,
+ OllamaToolCall,
+ OllamaToolCallFunction,
+)
+from litellm.types.llms.openai import (
+ AllMessageValues,
+ ChatCompletionAssistantToolCall,
+ ChatCompletionUsageBlock,
+)
+from litellm.types.utils import ModelResponse, ModelResponseStream
+
+from ..common_utils import OllamaError
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+ LiteLLMLoggingObj = Any
+
+
+class OllamaChatConfig(BaseConfig):
+ """
+ Reference: https://github.com/ollama/ollama/blob/main/docs/api.md#parameters
+
+ The class `OllamaConfig` provides the configuration for the Ollama's API interface. Below are the parameters:
+
+ - `mirostat` (int): Enable Mirostat sampling for controlling perplexity. Default is 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0. Example usage: mirostat 0
+
+ - `mirostat_eta` (float): Influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. Default: 0.1. Example usage: mirostat_eta 0.1
+
+ - `mirostat_tau` (float): Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. Default: 5.0. Example usage: mirostat_tau 5.0
+
+ - `num_ctx` (int): Sets the size of the context window used to generate the next token. Default: 2048. Example usage: num_ctx 4096
+
+ - `num_gqa` (int): The number of GQA groups in the transformer layer. Required for some models, for example it is 8 for llama2:70b. Example usage: num_gqa 1
+
+ - `num_gpu` (int): The number of layers to send to the GPU(s). On macOS it defaults to 1 to enable metal support, 0 to disable. Example usage: num_gpu 0
+
+ - `num_thread` (int): Sets the number of threads to use during computation. By default, Ollama will detect this for optimal performance. It is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). Example usage: num_thread 8
+
+ - `repeat_last_n` (int): Sets how far back for the model to look back to prevent repetition. Default: 64, 0 = disabled, -1 = num_ctx. Example usage: repeat_last_n 64
+
+ - `repeat_penalty` (float): Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. Default: 1.1. Example usage: repeat_penalty 1.1
+
+ - `temperature` (float): The temperature of the model. Increasing the temperature will make the model answer more creatively. Default: 0.8. Example usage: temperature 0.7
+
+ - `seed` (int): Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. Example usage: seed 42
+
+ - `stop` (string[]): Sets the stop sequences to use. Example usage: stop "AI assistant:"
+
+ - `tfs_z` (float): Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. Default: 1. Example usage: tfs_z 1
+
+ - `num_predict` (int): Maximum number of tokens to predict when generating text. Default: 128, -1 = infinite generation, -2 = fill context. Example usage: num_predict 42
+
+ - `top_k` (int): Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. Default: 40. Example usage: top_k 40
+
+ - `top_p` (float): Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. Default: 0.9. Example usage: top_p 0.9
+
+ - `system` (string): system prompt for model (overrides what is defined in the Modelfile)
+
+ - `template` (string): the full prompt or prompt template (overrides what is defined in the Modelfile)
+ """
+
+ mirostat: Optional[int] = None
+ mirostat_eta: Optional[float] = None
+ mirostat_tau: Optional[float] = None
+ num_ctx: Optional[int] = None
+ num_gqa: Optional[int] = None
+ num_thread: Optional[int] = None
+ repeat_last_n: Optional[int] = None
+ repeat_penalty: Optional[float] = None
+ seed: Optional[int] = None
+ tfs_z: Optional[float] = None
+ num_predict: Optional[int] = None
+ top_k: Optional[int] = None
+ system: Optional[str] = None
+ template: Optional[str] = None
+
+ def __init__(
+ self,
+ mirostat: Optional[int] = None,
+ mirostat_eta: Optional[float] = None,
+ mirostat_tau: Optional[float] = None,
+ num_ctx: Optional[int] = None,
+ num_gqa: Optional[int] = None,
+ num_thread: Optional[int] = None,
+ repeat_last_n: Optional[int] = None,
+ repeat_penalty: Optional[float] = None,
+ temperature: Optional[float] = None,
+ seed: Optional[int] = None,
+ stop: Optional[list] = None,
+ tfs_z: Optional[float] = None,
+ num_predict: Optional[int] = None,
+ top_k: Optional[int] = None,
+ top_p: Optional[float] = None,
+ system: Optional[str] = None,
+ template: Optional[str] = None,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return super().get_config()
+
+ def get_supported_openai_params(self, model: str):
+ return [
+ "max_tokens",
+ "max_completion_tokens",
+ "stream",
+ "top_p",
+ "temperature",
+ "seed",
+ "frequency_penalty",
+ "stop",
+ "tools",
+ "tool_choice",
+ "functions",
+ "response_format",
+ "reasoning_effort",
+ ]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ for param, value in non_default_params.items():
+ if param == "max_tokens" or param == "max_completion_tokens":
+ optional_params["num_predict"] = value
+ if param == "stream":
+ optional_params["stream"] = value
+ if param == "temperature":
+ optional_params["temperature"] = value
+ if param == "seed":
+ optional_params["seed"] = value
+ if param == "top_p":
+ optional_params["top_p"] = value
+ if param == "frequency_penalty":
+ optional_params["repeat_penalty"] = value
+ if param == "stop":
+ optional_params["stop"] = value
+ if (
+ param == "response_format"
+ and isinstance(value, dict)
+ and value.get("type") == "json_object"
+ ):
+ optional_params["format"] = "json"
+ if (
+ param == "response_format"
+ and isinstance(value, dict)
+ and value.get("type") == "json_schema"
+ ):
+ if value.get("json_schema") and value["json_schema"].get("schema"):
+ optional_params["format"] = value["json_schema"]["schema"]
+ if param == "reasoning_effort" and value is not None:
+ if model.startswith("gpt-oss"):
+ optional_params["think"] = value
+ else:
+ optional_params["think"] = value in {"low", "medium", "high"}
+ ### FUNCTION CALLING LOGIC ###
+ # Ollama 0.4+ supports native tool calling - pass tools directly
+ # and let Ollama handle model capability detection
+ # Fixes: https://github.com/BerriAI/litellm/issues/18922
+ if param == "tools":
+ optional_params["tools"] = value
+
+ if param == "functions":
+ optional_params["tools"] = value
+ non_default_params.pop("tool_choice", None) # causes ollama requests to hang
+ non_default_params.pop("functions", None) # causes ollama requests to hang
+ return optional_params
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ if api_key is not None and "Authorization" not in headers:
+ headers["Authorization"] = f"Bearer {api_key}"
+ return headers
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ api_key: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: dict,
+ stream: Optional[bool] = None,
+ ) -> str:
+ """
+ OPTIONAL
+
+ Get the complete url for the request
+
+ Some providers need `model` in `api_base`
+ """
+ if api_base is None:
+ api_base = "http://localhost:11434"
+ if api_base.endswith("/api/chat"):
+ url = api_base
+ else:
+ url = f"{api_base}/api/chat"
+
+ return url
+
+ def transform_request(
+ self,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ headers: dict,
+ ) -> dict:
+ stream = optional_params.pop("stream", False)
+ format = optional_params.pop("format", None)
+ keep_alive = optional_params.pop("keep_alive", None)
+ think = optional_params.pop("think", None)
+ function_name = optional_params.pop("function_name", None)
+ litellm_params["function_name"] = function_name
+ tools = optional_params.pop("tools", None)
+
+ new_messages = []
+ for m in messages:
+ if isinstance(
+ m, BaseModel
+ ): # avoid message serialization issues - https://github.com/BerriAI/litellm/issues/5319
+ m = m.model_dump(exclude_none=True)
+ tool_calls = m.get("tool_calls")
+ if tool_calls is not None and isinstance(tool_calls, list):
+ new_tools: List[OllamaToolCall] = []
+ for tool in tool_calls:
+ typed_tool = ChatCompletionAssistantToolCall(**tool) # type: ignore
+ if typed_tool["type"] == "function":
+ arguments = {}
+ if "arguments" in typed_tool["function"]:
+ arguments = json.loads(typed_tool["function"]["arguments"])
+ ollama_tool_call = OllamaToolCall(
+ function=OllamaToolCallFunction(
+ name=typed_tool["function"].get("name") or "",
+ arguments=arguments,
+ )
+ )
+ new_tools.append(ollama_tool_call)
+ cast(dict, m)["tool_calls"] = new_tools
+ reasoning_content, parsed_content = _extract_reasoning_content(
+ cast(dict, m)
+ )
+ content_str = convert_content_list_to_str(cast(AllMessageValues, m))
+ images = extract_images_from_message(cast(AllMessageValues, m))
+
+ ollama_message = OllamaChatCompletionMessage(
+ role=cast(str, m.get("role")),
+ )
+ if reasoning_content is not None:
+ ollama_message["thinking"] = reasoning_content
+ if content_str is not None:
+ ollama_message["content"] = content_str
+ if images is not None:
+ ollama_message["images"] = images
+
+ new_messages.append(ollama_message)
+
+ # Load Config
+ config = self.get_config()
+ for k, v in config.items():
+ if k not in optional_params:
+ optional_params[k] = v
+
+ data = {
+ "model": model,
+ "messages": new_messages,
+ "options": optional_params,
+ "stream": stream,
+ }
+ if format is not None:
+ data["format"] = format
+ if tools is not None:
+ data["tools"] = tools
+ if keep_alive is not None:
+ data["keep_alive"] = keep_alive
+ if think is not None:
+ data["think"] = think
+
+ return data
+
+ def transform_response(
+ self,
+ model: str,
+ raw_response: Response,
+ model_response: ModelResponse,
+ logging_obj: LiteLLMLoggingObj,
+ request_data: dict,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: str,
+ api_key: Optional[str] = None,
+ json_mode: Optional[bool] = None,
+ ) -> ModelResponse:
+ ## LOGGING
+ logging_obj.post_call(
+ input=messages,
+ api_key="",
+ original_response=raw_response.text,
+ additional_args={
+ "headers": None,
+ "api_base": litellm_params.get("api_base"),
+ },
+ )
+
+ response_json = raw_response.json()
+
+ ## RESPONSE OBJECT
+ model_response.choices[0].finish_reason = "stop"
+ response_json_message = response_json.get("message")
+ if response_json_message is not None:
+ if "thinking" in response_json_message:
+ # remap 'thinking' to 'reasoning_content'
+ response_json_message["reasoning_content"] = response_json_message[
+ "thinking"
+ ]
+ del response_json_message["thinking"]
+ elif response_json_message.get("content") is not None:
+ # parse reasoning content from content
+ from litellm.litellm_core_utils.prompt_templates.common_utils import (
+ _parse_content_for_reasoning,
+ )
+
+ reasoning_content, content = _parse_content_for_reasoning(
+ response_json_message["content"]
+ )
+ response_json_message["reasoning_content"] = reasoning_content
+ response_json_message["content"] = content
+
+ if (
+ request_data.get("format", "") == "json"
+ and litellm_params.get("function_name") is not None
+ ):
+ function_call = json.loads(response_json_message["content"])
+ message = litellm.Message(
+ content=None,
+ tool_calls=[
+ {
+ "id": f"call_{str(uuid.uuid4())}",
+ "function": {
+ "name": function_call.get(
+ "name", litellm_params.get("function_name")
+ ),
+ "arguments": json.dumps(
+ function_call.get("arguments", function_call)
+ ),
+ },
+ "type": "function",
+ }
+ ],
+ reasoning_content=response_json_message.get("reasoning_content"),
+ )
+ model_response.choices[0].message = message # type: ignore
+ model_response.choices[0].finish_reason = "tool_calls"
+ else:
+ _message = litellm.Message(**response_json_message)
+ model_response.choices[0].message = _message # type: ignore
+ # Set finish_reason to "tool_calls" when tool_calls are present
+ # Fixes: https://github.com/BerriAI/litellm/issues/18922
+ if _message.tool_calls:
+ model_response.choices[0].finish_reason = "tool_calls"
+ model_response.created = int(time.time())
+ model_response.model = "ollama_chat/" + model
+ prompt_tokens = response_json.get("prompt_eval_count", litellm.token_counter(messages=messages)) # type: ignore
+ completion_tokens = response_json.get(
+ "eval_count",
+ litellm.token_counter(text=response_json["message"]["content"]),
+ )
+ setattr(
+ model_response,
+ "usage",
+ litellm.Usage(
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ total_tokens=prompt_tokens + completion_tokens,
+ ),
+ )
+ return model_response
+
+ def get_error_class(
+ self, error_message: str, status_code: int, headers: Union[dict, Headers]
+ ) -> BaseLLMException:
+ return OllamaError(
+ status_code=status_code, message=error_message, headers=headers
+ )
+
+ def get_model_response_iterator(
+ self,
+ streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
+ sync_stream: bool,
+ json_mode: Optional[bool] = False,
+ ):
+ return OllamaChatCompletionResponseIterator(
+ streaming_response=streaming_response,
+ sync_stream=sync_stream,
+ json_mode=json_mode,
+ )
+
+
+class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
+ started_reasoning_content: bool = False
+ finished_reasoning_content: bool = False
+
+ def _is_function_call_complete(self, function_args: Union[str, dict]) -> bool:
+ if isinstance(function_args, dict):
+ return True
+ try:
+ json.loads(function_args)
+ return True
+ except Exception:
+ return False
+
+ def chunk_parser(self, chunk: dict) -> ModelResponseStream:
+ try:
+ """
+ Expected chunk format:
+ {
+ "model": "llama3.1",
+ "created_at": "2025-05-24T02:12:05.859654Z",
+ "message": {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [{
+ "function": {
+ "name": "get_latest_album_ratings",
+ "arguments": {
+ "artist_name": "Taylor Swift"
+ }
+ }
+ }]
+ },
+ "done_reason": "stop",
+ "done": true,
+ ...
+ }
+
+ Need to:
+ - convert 'message' to 'delta'
+ - return finish_reason when done is true
+ - return usage when done is true
+
+ """
+ from litellm.types.utils import Delta, StreamingChoices
+
+ # process tool calls - if complete function arg - add id to tool call
+ tool_calls = chunk["message"].get("tool_calls")
+ if tool_calls is not None:
+ for tool_call in tool_calls:
+ function_args = tool_call.get("function").get("arguments")
+ if function_args is not None and len(function_args) > 0:
+ is_function_call_complete = self._is_function_call_complete(
+ function_args
+ )
+ if is_function_call_complete:
+ tool_call["id"] = str(uuid.uuid4())
+
+ # PROCESS REASONING CONTENT
+ reasoning_content: Optional[str] = None
+ content: Optional[str] = None
+ if chunk["message"].get("thinking") is not None:
+ reasoning_content = chunk["message"].get("thinking")
+ self.started_reasoning_content = True
+ elif chunk["message"].get("content") is not None:
+ if (
+ self.started_reasoning_content
+ and not self.finished_reasoning_content
+ ):
+ self.finished_reasoning_content = True
+
+ message_content = chunk["message"].get("content")
+ if "" in message_content:
+ message_content = message_content.replace("", "")
+
+ self.started_reasoning_content = True
+
+ if "" in message_content and self.started_reasoning_content:
+ message_content = message_content.replace("", "")
+ self.finished_reasoning_content = True
+
+ if (
+ self.started_reasoning_content
+ and not self.finished_reasoning_content
+ ):
+ reasoning_content = message_content
+ else:
+ content = message_content
+
+ delta = Delta(
+ content=content,
+ reasoning_content=reasoning_content,
+ tool_calls=tool_calls,
+ )
+
+ if chunk["done"] is True:
+ finish_reason = chunk.get("done_reason", "stop")
+ # Override finish_reason when tool_calls are present
+ # Fixes: https://github.com/BerriAI/litellm/issues/18922
+ if tool_calls is not None:
+ finish_reason = "tool_calls"
+ choices = [
+ StreamingChoices(
+ delta=delta,
+ finish_reason=finish_reason,
+ )
+ ]
+ else:
+ choices = [
+ StreamingChoices(
+ delta=delta,
+ )
+ ]
+
+ usage = ChatCompletionUsageBlock(
+ prompt_tokens=chunk.get("prompt_eval_count", 0),
+ completion_tokens=chunk.get("eval_count", 0),
+ total_tokens=chunk.get("prompt_eval_count", 0)
+ + chunk.get("eval_count", 0),
+ )
+
+ return ModelResponseStream(
+ id=str(uuid.uuid4()),
+ object="chat.completion.chunk",
+ created=int(time.time()), # ollama created_at is in UTC
+ usage=usage,
+ model=chunk["model"],
+ choices=choices,
+ )
+ except KeyError as e:
+ raise OllamaError(
+ message=f"KeyError: {e}, Got unexpected response from Ollama: {chunk}",
+ status_code=400,
+ headers={"Content-Type": "application/json"},
+ )
+ except Exception as e:
+ raise e
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/common_utils.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/common_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..8aedd9b3500191bd7e4721b5e4a45cccba1d240d
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/common_utils.py
@@ -0,0 +1,149 @@
+from typing import List, Optional, Union
+
+import httpx
+
+from litellm import verbose_logger
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+
+class OllamaError(BaseLLMException):
+ def __init__(
+ self, status_code: int, message: str, headers: Union[dict, httpx.Headers]
+ ):
+ super().__init__(status_code=status_code, message=message, headers=headers)
+
+
+def _convert_image(image):
+ """
+ Convert image to base64 encoded image if not already in base64 format
+
+ If image is already in base64 format AND is a jpeg/png, return it
+
+ If image is not JPEG/PNG, convert it to JPEG base64 format
+ """
+ import base64
+ import io
+
+ try:
+ from PIL import Image
+ except Exception:
+ raise Exception(
+ "ollama image conversion failed please run `pip install Pillow`"
+ )
+
+ orig = image
+ if image.startswith("data:"):
+ image = image.split(",")[-1]
+ try:
+ image_data = Image.open(io.BytesIO(base64.b64decode(image)))
+ if image_data.format in ["JPEG", "PNG"]:
+ return image
+ except Exception:
+ return orig
+ jpeg_image = io.BytesIO()
+ image_data.convert("RGB").save(jpeg_image, "JPEG")
+ jpeg_image.seek(0)
+ return base64.b64encode(jpeg_image.getvalue()).decode("utf-8")
+
+
+from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
+
+
+class OllamaModelInfo(BaseLLMModelInfo):
+ """
+ Dynamic model listing for Ollama server.
+ Fetches /api/models and /api/tags, then for each tag also /api/models?tag=...
+ Returns the union of all model names.
+ """
+
+ @staticmethod
+ def get_api_key(api_key=None) -> Optional[str]:
+ """Get API key from environment variables or litellm configuration"""
+ import os
+
+ import litellm
+ from litellm.secret_managers.main import get_secret_str
+
+ return (
+ os.environ.get("OLLAMA_API_KEY")
+ or litellm.api_key
+ or litellm.openai_key
+ or get_secret_str("OLLAMA_API_KEY")
+ )
+
+ @staticmethod
+ def get_api_base(api_base: Optional[str] = None) -> str:
+ from litellm.secret_managers.main import get_secret_str
+
+ # env var OLLAMA_API_BASE or default
+ return api_base or get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434"
+
+ def get_models(self, api_key=None, api_base: Optional[str] = None) -> List[str]:
+ """
+ List all models available on the Ollama server via /api/tags endpoint.
+ """
+
+ base = self.get_api_base(api_base)
+ api_key = self.get_api_key()
+ headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+
+ names: set[str] = set()
+ try:
+ resp = httpx.get(f"{base}/api/tags", headers=headers)
+ resp.raise_for_status()
+ data = resp.json()
+ # Expecting a dict with a 'models' list
+ models_list = []
+ if (
+ isinstance(data, dict)
+ and "models" in data
+ and isinstance(data["models"], list)
+ ):
+ models_list = data["models"]
+ elif isinstance(data, list):
+ models_list = data
+ # Extract model names
+ for entry in models_list:
+ if not isinstance(entry, dict):
+ continue
+ nm = entry.get("name") or entry.get("model")
+ if isinstance(nm, str):
+ names.add(nm)
+ except Exception as e:
+ verbose_logger.warning(f"Error retrieving ollama tag endpoint: {e}")
+ # If tags endpoint fails, fall back to static list
+ try:
+ from litellm import models_by_provider
+
+ static = models_by_provider.get("ollama", []) or []
+ return [f"ollama/{m}" for m in static]
+ except Exception as e1:
+ verbose_logger.warning(
+ f"Error retrieving static ollama models as fallback: {e1}"
+ )
+ return []
+ # assemble full model names
+ result = sorted(names)
+ return result
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages: list,
+ optional_params: dict,
+ litellm_params: dict,
+ api_key=None,
+ api_base=None,
+ ) -> dict:
+ """
+ No-op environment validation for Ollama.
+ """
+ return {}
+
+ @staticmethod
+ def get_base_model(model: str) -> str:
+ """
+ Return the base model name for Ollama (no-op).
+ """
+ return model
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/completion/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/completion/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..97e4f13b5606e9fc95e557377faba19bfea2da68
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/completion/handler.py
@@ -0,0 +1,123 @@
+"""
+Ollama /chat/completion calls handled in llm_http_handler.py
+
+[TODO]: migrate embeddings to a base handler as well.
+"""
+
+from typing import Any, Dict, List
+
+import litellm
+from litellm.types.utils import EmbeddingResponse
+
+
+def _prepare_ollama_embedding_payload(
+ model: str, prompts: List[str], optional_params: Dict[str, Any]
+) -> Dict[str, Any]:
+ data: Dict[str, Any] = {"model": model, "input": prompts}
+ special_optional_params = ["truncate", "options", "keep_alive", "dimensions"]
+
+ for k, v in optional_params.items():
+ if k in special_optional_params:
+ data[k] = v
+ else:
+ data.setdefault("options", {})
+ if isinstance(data["options"], dict):
+ data["options"].update({k: v})
+ return data
+
+
+def _process_ollama_embedding_response(
+ response_json: dict,
+ prompts: List[str],
+ model: str,
+ model_response: EmbeddingResponse,
+ logging_obj: Any,
+ encoding: Any,
+) -> EmbeddingResponse:
+ output_data = []
+ embeddings: List[List[float]] = response_json["embeddings"]
+
+ for idx, emb in enumerate(embeddings):
+ output_data.append({"object": "embedding", "index": idx, "embedding": emb})
+
+ input_tokens = response_json.get("prompt_eval_count", None)
+
+ if input_tokens is None:
+ if encoding is not None:
+ input_tokens = len(encoding.encode("".join(prompts)))
+ if logging_obj:
+ logging_obj.debug(
+ "Ollama response missing prompt_eval_count; estimated with encoding."
+ )
+ else:
+ input_tokens = 0
+ if logging_obj:
+ logging_obj.warning(
+ "Missing prompt_eval_count and no encoding provided; defaulted to 0."
+ )
+
+ model_response.object = "list"
+ model_response.data = output_data
+ model_response.model = "ollama/" + model
+ model_response.usage = litellm.Usage(
+ prompt_tokens=input_tokens,
+ completion_tokens=0,
+ total_tokens=input_tokens,
+ prompt_tokens_details=None,
+ completion_tokens_details=None,
+ )
+ return model_response
+
+
+async def ollama_aembeddings(
+ api_base: str,
+ model: str,
+ prompts: List[str],
+ model_response: EmbeddingResponse,
+ optional_params: dict,
+ logging_obj: Any,
+ encoding: Any,
+):
+ if not api_base.endswith("/api/embed"):
+ api_base += "/api/embed"
+
+ data = _prepare_ollama_embedding_payload(model, prompts, optional_params)
+
+ response = await litellm.module_level_aclient.post(url=api_base, json=data)
+ response_json = response.json()
+
+ return _process_ollama_embedding_response(
+ response_json=response_json,
+ prompts=prompts,
+ model=model,
+ model_response=model_response,
+ logging_obj=logging_obj,
+ encoding=encoding,
+ )
+
+
+def ollama_embeddings(
+ api_base: str,
+ model: str,
+ prompts: List[str],
+ optional_params: dict,
+ model_response: EmbeddingResponse,
+ logging_obj: Any,
+ encoding: Any = None,
+):
+ if not api_base.endswith("/api/embed"):
+ api_base += "/api/embed"
+
+ data = _prepare_ollama_embedding_payload(model, prompts, optional_params)
+
+ response = litellm.module_level_client.post(url=api_base, json=data)
+ response_json = response.json()
+
+ return _process_ollama_embedding_response(
+ response_json=response_json,
+ prompts=prompts,
+ model=model,
+ model_response=model_response,
+ logging_obj=logging_obj,
+ encoding=encoding,
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/completion/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/completion/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..6a03325e6c7816d585afc3651615c8158b66cc69
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/ollama/completion/transformation.py
@@ -0,0 +1,612 @@
+import json
+import time
+from litellm._uuid import uuid
+from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional, Union
+
+from httpx._models import Headers, Response
+
+import litellm
+from litellm._logging import verbose_logger, verbose_proxy_logger
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+ get_str_from_messages,
+)
+from litellm.litellm_core_utils.prompt_templates.factory import (
+ convert_to_ollama_image,
+ custom_prompt,
+ ollama_pt,
+)
+from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
+from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock
+from litellm.types.utils import (
+ Delta,
+ GenericStreamingChunk,
+ ModelInfoBase,
+ ModelResponse,
+ ModelResponseStream,
+ ProviderField,
+ StreamingChoices,
+)
+
+from ..common_utils import OllamaError, _convert_image
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+ LiteLLMLoggingObj = Any
+
+
+class OllamaConfig(BaseConfig):
+ """
+ Reference: https://github.com/ollama/ollama/blob/main/docs/api.md#parameters
+
+ The class `OllamaConfig` provides the configuration for the Ollama's API interface. Below are the parameters:
+
+ - `mirostat` (int): Enable Mirostat sampling for controlling perplexity. Default is 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0. Example usage: mirostat 0
+
+ - `mirostat_eta` (float): Influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. Default: 0.1. Example usage: mirostat_eta 0.1
+
+ - `mirostat_tau` (float): Controls the balance between coherence and diversity of the output. A lower value will result in more focused and coherent text. Default: 5.0. Example usage: mirostat_tau 5.0
+
+ - `num_ctx` (int): Sets the size of the context window used to generate the next token. Default: 2048. Example usage: num_ctx 4096
+
+ - `num_gqa` (int): The number of GQA groups in the transformer layer. Required for some models, for example it is 8 for llama2:70b. Example usage: num_gqa 1
+
+ - `num_gpu` (int): The number of layers to send to the GPU(s). On macOS it defaults to 1 to enable metal support, 0 to disable. Example usage: num_gpu 0
+
+ - `num_thread` (int): Sets the number of threads to use during computation. By default, Ollama will detect this for optimal performance. It is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). Example usage: num_thread 8
+
+ - `repeat_last_n` (int): Sets how far back for the model to look back to prevent repetition. Default: 64, 0 = disabled, -1 = num_ctx. Example usage: repeat_last_n 64
+
+ - `repeat_penalty` (float): Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. Default: 1.1. Example usage: repeat_penalty 1.1
+
+ - `temperature` (float): The temperature of the model. Increasing the temperature will make the model answer more creatively. Default: 0.8. Example usage: temperature 0.7
+
+ - `seed` (int): Sets the random number seed to use for generation. Setting this to a specific number will make the model generate the same text for the same prompt. Example usage: seed 42
+
+ - `stop` (string[]): Sets the stop sequences to use. Example usage: stop "AI assistant:"
+
+ - `tfs_z` (float): Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting. Default: 1. Example usage: tfs_z 1
+
+ - `num_predict` (int): Maximum number of tokens to predict when generating text. Default: 128, -1 = infinite generation, -2 = fill context. Example usage: num_predict 42
+
+ - `top_k` (int): Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. Default: 40. Example usage: top_k 40
+
+ - `top_p` (float): Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. Default: 0.9. Example usage: top_p 0.9
+
+ - `system` (string): system prompt for model (overrides what is defined in the Modelfile)
+
+ - `template` (string): the full prompt or prompt template (overrides what is defined in the Modelfile)
+ """
+
+ mirostat: Optional[int] = None
+ mirostat_eta: Optional[float] = None
+ mirostat_tau: Optional[float] = None
+ num_ctx: Optional[int] = None
+ num_gqa: Optional[int] = None
+ num_gpu: Optional[int] = None
+ num_thread: Optional[int] = None
+ repeat_last_n: Optional[int] = None
+ repeat_penalty: Optional[float] = None
+ temperature: Optional[float] = None
+ seed: Optional[int] = None
+ stop: Optional[
+ list
+ ] = None # stop is a list based on this - https://github.com/ollama/ollama/pull/442
+ tfs_z: Optional[float] = None
+ num_predict: Optional[int] = None
+ top_k: Optional[int] = None
+ top_p: Optional[float] = None
+ system: Optional[str] = None
+ template: Optional[str] = None
+
+ def __init__(
+ self,
+ mirostat: Optional[int] = None,
+ mirostat_eta: Optional[float] = None,
+ mirostat_tau: Optional[float] = None,
+ num_ctx: Optional[int] = None,
+ num_gqa: Optional[int] = None,
+ num_gpu: Optional[int] = None,
+ num_thread: Optional[int] = None,
+ repeat_last_n: Optional[int] = None,
+ repeat_penalty: Optional[float] = None,
+ temperature: Optional[float] = None,
+ seed: Optional[int] = None,
+ stop: Optional[list] = None,
+ tfs_z: Optional[float] = None,
+ num_predict: Optional[int] = None,
+ top_k: Optional[int] = None,
+ top_p: Optional[float] = None,
+ system: Optional[str] = None,
+ template: Optional[str] = None,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return super().get_config()
+
+ def get_required_params(self) -> List[ProviderField]:
+ """For a given provider, return it's required fields with a description"""
+ return [
+ ProviderField(
+ field_name="base_url",
+ field_type="string",
+ field_description="Your Ollama API Base",
+ field_value="http://10.10.11.249:11434",
+ )
+ ]
+
+ def get_supported_openai_params(self, model: str):
+ return [
+ "max_tokens",
+ "stream",
+ "top_p",
+ "temperature",
+ "seed",
+ "frequency_penalty",
+ "stop",
+ "response_format",
+ "max_completion_tokens",
+ "reasoning_effort",
+ ]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ for param, value in non_default_params.items():
+ if param == "max_tokens" or param == "max_completion_tokens":
+ optional_params["num_predict"] = value
+ elif param == "stream":
+ optional_params["stream"] = value
+ elif param == "temperature":
+ optional_params["temperature"] = value
+ elif param == "seed":
+ optional_params["seed"] = value
+ elif param == "top_p":
+ optional_params["top_p"] = value
+ elif param == "frequency_penalty":
+ optional_params["frequency_penalty"] = value
+ elif param == "stop":
+ optional_params["stop"] = value
+ elif param == "reasoning_effort" and value is not None:
+ if model.startswith("gpt-oss"):
+ optional_params["think"] = value
+ else:
+ optional_params["think"] = value in {"low", "medium", "high"}
+ elif param == "response_format" and isinstance(value, dict):
+ if value["type"] == "json_object":
+ optional_params["format"] = "json"
+ elif value["type"] == "json_schema":
+ optional_params["format"] = value["json_schema"]["schema"]
+
+ return optional_params
+
+ def _supports_function_calling(self, ollama_model_info: dict) -> bool:
+ """
+ Check if the 'template' field in the ollama_model_info contains a 'tools' or 'function' key.
+ """
+ _template: str = str(ollama_model_info.get("template", "") or "")
+ return "tools" in _template.lower()
+
+ def _get_max_tokens(self, ollama_model_info: dict) -> Optional[int]:
+ _model_info: dict = ollama_model_info.get("model_info", {})
+
+ for k, v in _model_info.items():
+ if "context_length" in k:
+ return v
+ return None
+
+ @staticmethod
+ def get_api_key() -> Optional[str]:
+ """Get API key from environment variables or litellm configuration"""
+ import os
+
+ import litellm
+ from litellm.secret_managers.main import get_secret_str
+
+ return (
+ os.environ.get("OLLAMA_API_KEY")
+ or litellm.api_key
+ or litellm.openai_key
+ or get_secret_str("OLLAMA_API_KEY")
+ )
+
+ def get_model_info(
+ self, model: str, api_base: Optional[str] = None
+ ) -> ModelInfoBase:
+ """
+ curl http://localhost:11434/api/show -d '{
+ "name": "mistral"
+ }'
+ """
+ if model.startswith("ollama/") or model.startswith("ollama_chat/"):
+ model = model.split("/", 1)[1]
+ api_base = (
+ api_base or get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434"
+ )
+ api_key = self.get_api_key()
+ headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+
+ try:
+ response = litellm.module_level_client.post(
+ url=f"{api_base}/api/show",
+ json={"name": model},
+ headers=headers,
+ )
+ except Exception as e:
+ verbose_logger.debug(
+ "OllamaError: Could not get model info for %s from %s. Error: %s",
+ model,
+ api_base,
+ e,
+ )
+ return ModelInfoBase(
+ key=model,
+ litellm_provider="ollama",
+ mode="chat",
+ input_cost_per_token=0.0,
+ output_cost_per_token=0.0,
+ max_tokens=None,
+ max_input_tokens=None,
+ max_output_tokens=None,
+ )
+
+ model_info = response.json()
+
+ _max_tokens: Optional[int] = self._get_max_tokens(model_info)
+
+ return ModelInfoBase(
+ key=model,
+ litellm_provider="ollama",
+ mode="chat",
+ supports_function_calling=self._supports_function_calling(model_info),
+ input_cost_per_token=0.0,
+ output_cost_per_token=0.0,
+ max_tokens=_max_tokens,
+ max_input_tokens=_max_tokens,
+ max_output_tokens=_max_tokens,
+ )
+
+ def get_error_class(
+ self, error_message: str, status_code: int, headers: Union[dict, Headers]
+ ) -> BaseLLMException:
+ return OllamaError(
+ status_code=status_code, message=error_message, headers=headers
+ )
+
+ def transform_response(
+ self,
+ model: str,
+ raw_response: Response,
+ model_response: ModelResponse,
+ logging_obj: LiteLLMLoggingObj,
+ request_data: dict,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: str,
+ api_key: Optional[str] = None,
+ json_mode: Optional[bool] = None,
+ ) -> ModelResponse:
+ from litellm.litellm_core_utils.prompt_templates.common_utils import (
+ _parse_content_for_reasoning,
+ )
+
+ response_json = raw_response.json()
+ ## RESPONSE OBJECT
+ model_response.choices[0].finish_reason = "stop"
+ if request_data.get("format", "") == "json":
+ # Check if response field exists and is not empty before parsing JSON
+ response_text = response_json.get("response", "")
+
+ if not response_text or not response_text.strip():
+ # Handle empty response gracefully - set empty content
+ message = litellm.Message(content="")
+ model_response.choices[0].message = message # type: ignore
+ model_response.choices[0].finish_reason = "stop"
+ else:
+ try:
+ response_content = json.loads(response_text)
+
+ # Check if this is a function call format with name/arguments structure
+ if (
+ isinstance(response_content, dict)
+ and "name" in response_content
+ and "arguments" in response_content
+ ):
+ # Handle as function call (original behavior)
+ function_call = response_content
+ message = litellm.Message(
+ content=None,
+ tool_calls=[
+ {
+ "id": f"call_{str(uuid.uuid4())}",
+ "function": {
+ "name": function_call["name"],
+ "arguments": json.dumps(
+ function_call["arguments"]
+ ),
+ },
+ "type": "function",
+ }
+ ],
+ )
+ model_response.choices[0].message = message # type: ignore
+ model_response.choices[0].finish_reason = "tool_calls"
+ else:
+ # Handle as regular JSON (new behavior)
+ message = litellm.Message(
+ content=json.dumps(response_content),
+ )
+ model_response.choices[0].message = message # type: ignore
+ model_response.choices[0].finish_reason = "stop"
+ except json.JSONDecodeError:
+ # If JSON parsing fails, treat as regular text response
+ ## output parse reasoning content from response_text
+ reasoning_content: Optional[str] = None
+ content: Optional[str] = None
+ if response_text is not None:
+ reasoning_content, content = _parse_content_for_reasoning(
+ response_text
+ )
+ message = litellm.Message(
+ content=content, reasoning_content=reasoning_content
+ )
+ model_response.choices[0].message = message # type: ignore
+ model_response.choices[0].finish_reason = "stop"
+ else:
+ response_text = response_json.get("response", "")
+ content = None
+ reasoning_content = None
+ if response_text is not None and isinstance(response_text, str):
+ reasoning_content, content = _parse_content_for_reasoning(response_text)
+ else:
+ content = response_text # type: ignore
+ model_response.choices[0].message.content = content # type: ignore
+ model_response.choices[0].message.reasoning_content = reasoning_content # type: ignore
+ model_response.created = int(time.time())
+ model_response.model = "ollama/" + model
+ _prompt = request_data.get("prompt", "")
+ prompt_tokens = response_json.get(
+ "prompt_eval_count", len(encoding.encode(_prompt, disallowed_special=())) # type: ignore
+ )
+ completion_tokens = response_json.get(
+ "eval_count", len(response_json.get("message", dict()).get("content", ""))
+ )
+ setattr(
+ model_response,
+ "usage",
+ litellm.Usage(
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ total_tokens=prompt_tokens + completion_tokens,
+ ),
+ )
+ return model_response
+
+ def transform_request(
+ self,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ headers: dict,
+ ) -> dict:
+ custom_prompt_dict = (
+ litellm_params.get("custom_prompt_dict") or litellm.custom_prompt_dict
+ )
+
+ text_completion_request = litellm_params.get("text_completion")
+ if model in custom_prompt_dict:
+ # check if the model has a registered custom prompt
+ model_prompt_details = custom_prompt_dict[model]
+ ollama_prompt = custom_prompt(
+ role_dict=model_prompt_details["roles"],
+ initial_prompt_value=model_prompt_details["initial_prompt_value"],
+ final_prompt_value=model_prompt_details["final_prompt_value"],
+ messages=messages,
+ )
+ elif text_completion_request: # handle `/completions` requests
+ ollama_prompt = get_str_from_messages(messages=messages)
+ else: # handle `/chat/completions` requests
+ modified_prompt = ollama_pt(model=model, messages=messages)
+ if isinstance(modified_prompt, dict):
+ ollama_prompt, images = (
+ modified_prompt["prompt"],
+ modified_prompt["images"],
+ )
+ optional_params["images"] = images
+ else:
+ ollama_prompt = modified_prompt
+ stream = optional_params.pop("stream", False)
+ format = optional_params.pop("format", None)
+ images = optional_params.pop("images", None)
+ think = optional_params.pop("think", None)
+ data = {
+ "model": model,
+ "prompt": ollama_prompt,
+ "options": optional_params,
+ "stream": stream,
+ }
+
+ if format is not None:
+ data["format"] = format
+ if images is not None:
+ data["images"] = [
+ _convert_image(convert_to_ollama_image(image)) for image in images
+ ]
+ if think is not None:
+ data["think"] = think
+
+ return data
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ return headers
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ api_key: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: dict,
+ stream: Optional[bool] = None,
+ ) -> str:
+ """
+ OPTIONAL
+
+ Get the complete url for the request
+
+ Some providers need `model` in `api_base`
+ """
+ if api_base is None:
+ api_base = "http://localhost:11434"
+ if api_base.endswith("/api/generate"):
+ url = api_base
+ else:
+ url = f"{api_base}/api/generate"
+
+ return url
+
+ def get_model_response_iterator(
+ self,
+ streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
+ sync_stream: bool,
+ json_mode: Optional[bool] = False,
+ ):
+ return OllamaTextCompletionResponseIterator(
+ streaming_response=streaming_response,
+ sync_stream=sync_stream,
+ json_mode=json_mode,
+ )
+
+
+class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
+ def __init__(
+ self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False
+ ):
+ super().__init__(streaming_response, sync_stream, json_mode)
+ self.started_reasoning_content: bool = False
+ self.finished_reasoning_content: bool = False
+
+ def _handle_string_chunk(
+ self, str_line: str
+ ) -> Union[GenericStreamingChunk, ModelResponseStream]:
+ return self.chunk_parser(json.loads(str_line))
+
+ def chunk_parser(
+ self, chunk: dict
+ ) -> Union[GenericStreamingChunk, ModelResponseStream]:
+ try:
+ if "error" in chunk:
+ raise Exception(f"Ollama Error - {chunk}")
+
+ text = ""
+ is_finished = False
+ finish_reason = None
+ if chunk["done"] is True:
+ text = ""
+ is_finished = True
+ finish_reason = "stop"
+ prompt_eval_count: Optional[int] = chunk.get("prompt_eval_count", None)
+ eval_count: Optional[int] = chunk.get("eval_count", None)
+
+ usage: Optional[ChatCompletionUsageBlock] = None
+ if prompt_eval_count is not None and eval_count is not None:
+ usage = ChatCompletionUsageBlock(
+ prompt_tokens=prompt_eval_count,
+ completion_tokens=eval_count,
+ total_tokens=prompt_eval_count + eval_count,
+ )
+ return GenericStreamingChunk(
+ text=text,
+ is_finished=is_finished,
+ finish_reason=finish_reason,
+ usage=usage,
+ )
+ elif chunk["response"]:
+ text = chunk["response"]
+ reasoning_content: Optional[str] = None
+ content: Optional[str] = None
+ if text is not None:
+ if "" in text:
+ text = text.replace("", "")
+ self.started_reasoning_content = True
+ elif "" in text:
+ text = text.replace("", "")
+ self.finished_reasoning_content = True
+
+ if (
+ self.started_reasoning_content
+ and not self.finished_reasoning_content
+ ):
+ reasoning_content = text
+ else:
+ content = text
+
+ return ModelResponseStream(
+ choices=[
+ StreamingChoices(
+ index=0,
+ delta=Delta(
+ reasoning_content=reasoning_content, content=content
+ ),
+ )
+ ],
+ finish_reason=finish_reason,
+ usage=None,
+ )
+ # return GenericStreamingChunk(
+ # text=text,
+ # is_finished=is_finished,
+ # finish_reason="stop",
+ # usage=None,
+ # )
+ elif "thinking" in chunk and not chunk["response"]:
+ # Return reasoning content as ModelResponseStream so UIs can render it
+ thinking_content = chunk.get("thinking") or ""
+ return ModelResponseStream(
+ choices=[
+ StreamingChoices(
+ index=0,
+ delta=Delta(reasoning_content=thinking_content),
+ )
+ ]
+ )
+ else:
+ # In this case, 'thinking' is not present in the chunk, chunk["done"] is false,
+ # and chunk["response"] is falsy (None or empty string),
+ # but Ollama is just starting to stream, so it should be processed as a normal dict
+ return ModelResponseStream(
+ choices=[
+ StreamingChoices(
+ index=0,
+ delta=Delta(reasoning_content=""),
+ )
+ ]
+ )
+ # raise Exception(f"Unable to parse ollama chunk - {chunk}")
+ except Exception as e:
+ verbose_proxy_logger.error(f"Unable to parse ollama chunk - {chunk}")
+ raise e
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oobabooga/chat/oobabooga.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oobabooga/chat/oobabooga.py
new file mode 100644
index 0000000000000000000000000000000000000000..5eb68a03d4bdbce3292a397b0fd4095f9b5f0823
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oobabooga/chat/oobabooga.py
@@ -0,0 +1,160 @@
+import json
+from typing import Any, Callable, Optional
+
+import litellm
+from litellm.llms.custom_httpx.http_handler import _get_httpx_client
+from litellm.utils import EmbeddingResponse, ModelResponse, Usage
+
+from ..common_utils import OobaboogaError
+from .transformation import OobaboogaConfig
+
+oobabooga_config = OobaboogaConfig()
+
+
+def completion(
+ model: str,
+ messages: list,
+ api_base: Optional[str],
+ model_response: ModelResponse,
+ print_verbose: Callable,
+ encoding,
+ api_key,
+ logging_obj,
+ optional_params: dict,
+ litellm_params: dict,
+ custom_prompt_dict={},
+ logger_fn=None,
+ default_max_tokens_to_sample=None,
+):
+ headers = oobabooga_config.validate_environment(
+ api_key=api_key,
+ headers={},
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ )
+ if "https" in model:
+ completion_url = model
+ elif api_base:
+ completion_url = api_base
+ else:
+ raise OobaboogaError(
+ status_code=404,
+ message="API Base not set. Set one via completion(..,api_base='your-api-url')",
+ )
+ model = model
+
+ completion_url = completion_url + "/v1/chat/completions"
+ data = oobabooga_config.transform_request(
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ headers=headers,
+ )
+ ## LOGGING
+
+ logging_obj.pre_call(
+ input=messages,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data},
+ )
+ ## COMPLETION CALL
+ client = _get_httpx_client()
+ response = client.post(
+ completion_url,
+ headers=headers,
+ data=json.dumps(data),
+ stream=optional_params["stream"] if "stream" in optional_params else False,
+ )
+ if "stream" in optional_params and optional_params["stream"] is True:
+ return response.iter_lines()
+ else:
+ return oobabooga_config.transform_response(
+ model=model,
+ raw_response=response,
+ model_response=model_response,
+ logging_obj=logging_obj,
+ api_key=api_key,
+ request_data=data,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ encoding=encoding,
+ )
+
+
+def embedding(
+ model: str,
+ input: list,
+ model_response: EmbeddingResponse,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ logging_obj: Any,
+ optional_params: dict,
+ encoding=None,
+):
+ # Create completion URL
+ if "https" in model:
+ embeddings_url = model
+ elif api_base:
+ embeddings_url = f"{api_base}/v1/embeddings"
+ else:
+ raise OobaboogaError(
+ status_code=404,
+ message="API Base not set. Set one via completion(..,api_base='your-api-url')",
+ )
+
+ # Prepare request data
+ data = {"input": input}
+ if optional_params:
+ data.update(optional_params)
+
+ # Logging before API call
+ if logging_obj:
+ logging_obj.pre_call(
+ input=input, api_key=api_key, additional_args={"complete_input_dict": data}
+ )
+
+ # Send POST request
+ headers = oobabooga_config.validate_environment(
+ api_key=api_key,
+ headers={},
+ model=model,
+ messages=[],
+ optional_params=optional_params,
+ litellm_params={},
+ )
+ response = litellm.module_level_client.post(
+ embeddings_url, headers=headers, json=data
+ )
+ completion_response = response.json()
+
+ # Check for errors in response
+ if "error" in completion_response:
+ raise OobaboogaError(
+ message=completion_response["error"],
+ status_code=completion_response.get("status_code", 500),
+ )
+
+ # Process response data
+ model_response.data = [
+ {
+ "embedding": completion_response["data"][0]["embedding"],
+ "index": 0,
+ "object": "embedding",
+ }
+ ]
+
+ num_tokens = len(completion_response["data"][0]["embedding"])
+ # Adding metadata to response
+ setattr(
+ model_response,
+ "usage",
+ Usage(prompt_tokens=num_tokens, total_tokens=num_tokens),
+ )
+ model_response.object = "list"
+ model_response.model = model
+
+ return model_response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oobabooga/chat/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oobabooga/chat/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..e87b70130ce0bc3ce52b8296106bf9fa63daa351
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oobabooga/chat/transformation.py
@@ -0,0 +1,101 @@
+import time
+from typing import TYPE_CHECKING, Any, List, Optional, Union
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+from litellm.types.llms.openai import AllMessageValues
+from litellm.types.utils import ModelResponse, Usage
+
+from ..common_utils import OobaboogaError
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+ LoggingClass = LiteLLMLoggingObj
+else:
+ LoggingClass = Any
+
+
+class OobaboogaConfig(OpenAIGPTConfig):
+ def get_error_class(
+ self,
+ error_message: str,
+ status_code: int,
+ headers: Optional[Union[dict, httpx.Headers]] = None,
+ ) -> BaseLLMException:
+ return OobaboogaError(
+ status_code=status_code, message=error_message, headers=headers
+ )
+
+ def transform_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ model_response: ModelResponse,
+ logging_obj: LoggingClass,
+ request_data: dict,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: Any,
+ api_key: Optional[str] = None,
+ json_mode: Optional[bool] = None,
+ ) -> ModelResponse:
+ ## LOGGING
+ logging_obj.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=raw_response.text,
+ additional_args={"complete_input_dict": request_data},
+ )
+
+ ## RESPONSE OBJECT
+ try:
+ completion_response = raw_response.json()
+ except Exception:
+ raise OobaboogaError(
+ message=raw_response.text, status_code=raw_response.status_code
+ )
+ if "error" in completion_response:
+ raise OobaboogaError(
+ message=completion_response["error"],
+ status_code=raw_response.status_code,
+ )
+ else:
+ try:
+ model_response.choices[0].message.content = completion_response["choices"][0]["message"]["content"] # type: ignore
+ except Exception as e:
+ raise OobaboogaError(
+ message=str(e),
+ status_code=raw_response.status_code,
+ )
+
+ model_response.created = int(time.time())
+ model_response.model = model
+ usage = Usage(
+ prompt_tokens=completion_response["usage"]["prompt_tokens"],
+ completion_tokens=completion_response["usage"]["completion_tokens"],
+ total_tokens=completion_response["usage"]["total_tokens"],
+ )
+ setattr(model_response, "usage", usage)
+ return model_response
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ headers = {
+ "accept": "application/json",
+ "content-type": "application/json",
+ }
+ if api_key is not None:
+ headers["Authorization"] = f"Token {api_key}"
+ return headers
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oobabooga/common_utils.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oobabooga/common_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..82f8cda95117fbf9ec1cd05ddf18f05cd517138e
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/oobabooga/common_utils.py
@@ -0,0 +1,15 @@
+from typing import Optional, Union
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+
+class OobaboogaError(BaseLLMException):
+ def __init__(
+ self,
+ status_code: int,
+ message: str,
+ headers: Optional[Union[dict, httpx.Headers]] = None,
+ ):
+ super().__init__(status_code=status_code, message=message, headers=headers)
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/gpt_5_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/gpt_5_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..fc48704cd10bf2e054680545eeb46a4afab1b794
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/gpt_5_transformation.py
@@ -0,0 +1,307 @@
+"""Support for OpenAI gpt-5 model family."""
+
+from typing import Optional, Union
+
+import litellm
+from litellm.utils import _is_explicitly_disabled_factory, _supports_factory
+
+from .gpt_transformation import OpenAIGPTConfig
+
+
+def _normalize_reasoning_effort_for_chat_completion(
+ value: Union[str, dict, None],
+) -> Optional[str]:
+ """Convert reasoning_effort to the string format expected by OpenAI chat completion API.
+
+ The chat completion API expects a simple string: 'none', 'low', 'medium', 'high', or 'xhigh'.
+ Config/deployments may pass the Responses API format: {'effort': 'high', 'summary': 'detailed'}.
+ """
+ if value is None:
+ return None
+ if isinstance(value, str):
+ return value
+ if isinstance(value, dict) and "effort" in value:
+ return value["effort"]
+ return None
+
+
+def _get_effort_level(value: Union[str, dict, None]) -> Optional[str]:
+ """Extract the effective effort level from reasoning_effort (string or dict).
+
+ Use this for guards that compare effort level (e.g. xhigh validation, "none" checks).
+ Ensures dict inputs like {"effort": "none", "summary": "detailed"} are correctly
+ treated as effort="none" for validation purposes.
+ """
+ if value is None:
+ return None
+ if isinstance(value, str):
+ return value
+ if isinstance(value, dict) and "effort" in value:
+ return value["effort"]
+ return None
+
+
+class OpenAIGPT5Config(OpenAIGPTConfig):
+ """Configuration for gpt-5 models including GPT-5-Codex variants.
+
+ Handles OpenAI API quirks for the gpt-5 series like:
+
+ - Mapping ``max_tokens`` -> ``max_completion_tokens``.
+ - Dropping unsupported ``temperature`` values when requested.
+ - Support for GPT-5-Codex models optimized for code generation.
+ """
+
+ @classmethod
+ def is_model_gpt_5_model(cls, model: str) -> bool:
+ # gpt-5-chat* behaves like a regular chat model (supports temperature, etc.)
+ # Don't route it through GPT-5 reasoning-specific parameter restrictions.
+ return "gpt-5" in model and "gpt-5-chat" not in model
+
+ @classmethod
+ def is_model_gpt_5_search_model(cls, model: str) -> bool:
+ """Check if the model is a GPT-5 search variant (e.g. gpt-5-search-api).
+
+ Search-only models have a severely restricted parameter set compared to
+ regular GPT-5 models. They are identified by name convention (contain
+ both ``gpt-5`` and ``search``). Note: ``supports_web_search`` in model
+ info is a *different* concept — it indicates a model can *use* web
+ search as a tool, which many non-search-only models also support.
+ """
+ return "gpt-5" in model and "search" in model
+
+ @classmethod
+ def is_model_gpt_5_codex_model(cls, model: str) -> bool:
+ """Check if the model is specifically a GPT-5 Codex variant."""
+ return "gpt-5-codex" in model
+
+ @classmethod
+ def is_model_gpt_5_2_model(cls, model: str) -> bool:
+ """Check if the model is a gpt-5.2 variant (including pro)."""
+ model_name = model.split("/")[-1]
+ return model_name.startswith("gpt-5.2") or model_name.startswith("gpt-5.4")
+
+ @classmethod
+ def is_model_gpt_5_4_model(cls, model: str) -> bool:
+ """Check if the model is a gpt-5.4 variant (including pro)."""
+ model_name = model.split("/")[-1]
+ return model_name.startswith("gpt-5.4")
+
+ @classmethod
+ def is_model_gpt_5_4_plus_model(cls, model: str) -> bool:
+ """Check if the model is gpt-5.4 or newer (5.4, 5.5, 5.6, etc., including pro)."""
+ model_name = model.split("/")[-1]
+ if not model_name.startswith("gpt-5."):
+ return False
+ try:
+ version_str = model_name.replace("gpt-5.", "").split("-")[0]
+ major = version_str.split(".")[0]
+ return int(major) >= 4
+ except (ValueError, IndexError):
+ return False
+
+ @classmethod
+ def _supports_reasoning_effort_level(cls, model: str, level: str) -> bool:
+ """Check if the model supports a specific reasoning_effort level.
+
+ Looks up ``supports_{level}_reasoning_effort`` in the model map via
+ the shared ``_supports_factory`` helper.
+ Returns False for unknown models (safe fallback).
+ """
+ return _supports_factory(
+ model=model,
+ custom_llm_provider=None,
+ key=f"supports_{level}_reasoning_effort",
+ )
+
+ @classmethod
+ def _is_reasoning_effort_level_explicitly_disabled(
+ cls, model: str, level: str
+ ) -> bool:
+ """Return True only when the model map explicitly sets the capability to False.
+
+ Unlike ``_supports_reasoning_effort_level`` (which requires an explicit True),
+ this method returns True only when ``supports_{level}_reasoning_effort`` is
+ explicitly set to ``False`` in the model map. A missing key is treated as
+ supported (i.e. this method returns False = not disabled).
+
+ Use this for opt-out checks where unknown models should be allowed through.
+ """
+ return _is_explicitly_disabled_factory(
+ model=model,
+ custom_llm_provider=None,
+ key=f"supports_{level}_reasoning_effort",
+ )
+
+ def get_supported_openai_params(self, model: str) -> list:
+ if self.is_model_gpt_5_search_model(model):
+ return [
+ "max_tokens",
+ "max_completion_tokens",
+ "stream",
+ "stream_options",
+ "web_search_options",
+ "service_tier",
+ "safety_identifier",
+ "response_format",
+ "user",
+ "store",
+ "verbosity",
+ "max_retries",
+ "extra_headers",
+ ]
+
+ from litellm.utils import supports_tool_choice
+
+ base_gpt_series_params = super().get_supported_openai_params(model=model)
+ gpt_5_only_params = ["reasoning_effort", "verbosity"]
+ base_gpt_series_params.extend(gpt_5_only_params)
+ if not supports_tool_choice(model=model):
+ base_gpt_series_params.remove("tool_choice")
+
+ non_supported_params = [
+ "presence_penalty",
+ "frequency_penalty",
+ "stop",
+ "logit_bias",
+ "modalities",
+ "prediction",
+ "audio",
+ "web_search_options",
+ ]
+
+ # gpt-5.1/5.2 support logprobs, top_p, top_logprobs when reasoning_effort="none"
+ if not self._supports_reasoning_effort_level(model, "none"):
+ non_supported_params.extend(["logprobs", "top_p", "top_logprobs"])
+
+ return [
+ param
+ for param in base_gpt_series_params
+ if param not in non_supported_params
+ ]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ if self.is_model_gpt_5_search_model(model):
+ if "max_tokens" in non_default_params:
+ optional_params["max_completion_tokens"] = non_default_params.pop(
+ "max_tokens"
+ )
+ return super()._map_openai_params(
+ non_default_params=non_default_params,
+ optional_params=optional_params,
+ model=model,
+ drop_params=drop_params,
+ )
+
+ # Get raw reasoning_effort and effective effort level for all guards.
+ # Use effective_effort (extracted string) for xhigh validation, "none" checks, and
+ # tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"}
+ # must be treated as effort="none" to avoid incorrect tool-drop or sampling errors.
+ raw_reasoning_effort = non_default_params.get(
+ "reasoning_effort"
+ ) or optional_params.get("reasoning_effort")
+ effective_effort = _get_effort_level(raw_reasoning_effort)
+
+ # Normalize dict reasoning_effort to string for Chat Completions API.
+ # Example: {"effort": "high", "summary": "detailed"} -> "high"
+ if isinstance(raw_reasoning_effort, dict) and "effort" in raw_reasoning_effort:
+ normalized = _normalize_reasoning_effort_for_chat_completion(
+ raw_reasoning_effort
+ )
+ if normalized is not None:
+ if "reasoning_effort" in non_default_params:
+ non_default_params["reasoning_effort"] = normalized
+ if "reasoning_effort" in optional_params:
+ optional_params["reasoning_effort"] = normalized
+
+ if effective_effort == "xhigh":
+ # xhigh is an opt-in capability: only allow if model explicitly supports it.
+ if not self._supports_reasoning_effort_level(model, effective_effort):
+ if litellm.drop_params or drop_params:
+ non_default_params.pop("reasoning_effort", None)
+ optional_params.pop("reasoning_effort", None)
+ else:
+ raise litellm.utils.UnsupportedParamsError(
+ message=(
+ f"reasoning_effort={effective_effort} is not supported for this model."
+ ),
+ status_code=400,
+ )
+ elif effective_effort == "minimal":
+ # minimal is opt-out: unknown models pass through; only block when
+ # the model map explicitly sets supports_minimal_reasoning_effort=false.
+ if self._is_reasoning_effort_level_explicitly_disabled(
+ model, effective_effort
+ ):
+ if litellm.drop_params or drop_params:
+ non_default_params.pop("reasoning_effort", None)
+ optional_params.pop("reasoning_effort", None)
+ else:
+ raise litellm.utils.UnsupportedParamsError(
+ message=(
+ f"reasoning_effort={effective_effort} is not supported for this model."
+ ),
+ status_code=400,
+ )
+
+ ################################################################
+ # max_tokens is not supported for gpt-5 models on OpenAI API
+ # Relevant issue: https://github.com/BerriAI/litellm/issues/13381
+ ################################################################
+ if "max_tokens" in non_default_params:
+ optional_params["max_completion_tokens"] = non_default_params.pop(
+ "max_tokens"
+ )
+
+ # gpt-5.1/5.2 support logprobs, top_p, top_logprobs only when reasoning_effort="none"
+ supports_none = self._supports_reasoning_effort_level(model, "none")
+ if supports_none:
+ sampling_params = ["logprobs", "top_logprobs", "top_p"]
+ has_sampling = any(p in non_default_params for p in sampling_params)
+ if has_sampling and effective_effort not in (None, "none"):
+ if litellm.drop_params or drop_params:
+ for p in sampling_params:
+ non_default_params.pop(p, None)
+ else:
+ raise litellm.utils.UnsupportedParamsError(
+ message=(
+ "gpt-5.1/5.2/5.4 only support logprobs, top_p, top_logprobs when "
+ "reasoning_effort='none'. Current reasoning_effort='{}'. "
+ "To drop unsupported params set `litellm.drop_params = True`"
+ ).format(effective_effort),
+ status_code=400,
+ )
+
+ if "temperature" in non_default_params:
+ temperature_value: Optional[float] = non_default_params.pop("temperature")
+ if temperature_value is not None:
+ # models supporting reasoning_effort="none" also support flexible temperature
+ if supports_none and (
+ effective_effort == "none" or effective_effort is None
+ ):
+ optional_params["temperature"] = temperature_value
+ elif temperature_value == 1:
+ optional_params["temperature"] = temperature_value
+ elif litellm.drop_params or drop_params:
+ pass
+ else:
+ raise litellm.utils.UnsupportedParamsError(
+ message=(
+ "gpt-5 models (including gpt-5-codex) don't support temperature={}. "
+ "Only temperature=1 is supported. "
+ "For gpt-5.1, temperature is supported when reasoning_effort='none' (or not specified, as it defaults to 'none'). "
+ "To drop unsupported params set `litellm.drop_params = True`"
+ ).format(temperature_value),
+ status_code=400,
+ )
+ return super()._map_openai_params(
+ non_default_params=non_default_params,
+ optional_params=optional_params,
+ model=model,
+ drop_params=drop_params,
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/gpt_audio_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/gpt_audio_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..581ffea2db484803fc5eb92cb6688d4db6889a85
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/gpt_audio_transformation.py
@@ -0,0 +1,48 @@
+"""
+Support for GPT-4o audio Family
+
+OpenAI Doc: https://platform.openai.com/docs/guides/audio/quickstart?audio-generation-quickstart-example=audio-in&lang=python
+"""
+
+import litellm
+
+from .gpt_transformation import OpenAIGPTConfig
+
+
+class OpenAIGPTAudioConfig(OpenAIGPTConfig):
+ """
+ Reference: https://platform.openai.com/docs/guides/audio
+ """
+
+ @classmethod
+ def get_config(cls):
+ return super().get_config()
+
+ def get_supported_openai_params(self, model: str) -> list:
+ """
+ Get the supported OpenAI params for the `gpt-audio` models
+
+ """
+
+ all_openai_params = super().get_supported_openai_params(model=model)
+ audio_specific_params = ["audio"]
+ return all_openai_params + audio_specific_params
+
+ def is_model_gpt_audio_model(self, model: str) -> bool:
+ if model in litellm.open_ai_chat_completion_models and "audio" in model:
+ return True
+ return False
+
+ def _map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ return super()._map_openai_params(
+ non_default_params=non_default_params,
+ optional_params=optional_params,
+ model=model,
+ drop_params=drop_params,
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/gpt_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/gpt_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..c12c6e6ba09409af9029430fbfc55efe3adad80b
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/gpt_transformation.py
@@ -0,0 +1,820 @@
+"""
+Support for gpt model family
+"""
+
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ AsyncIterator,
+ Coroutine,
+ Dict,
+ Iterator,
+ List,
+ Literal,
+ Optional,
+ Tuple,
+ Union,
+ cast,
+ overload,
+)
+
+import httpx
+
+import litellm
+from litellm.litellm_core_utils.core_helpers import map_finish_reason
+from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+ _extract_reasoning_content,
+ _handle_invalid_parallel_tool_calls,
+ _should_convert_tool_call_to_json_mode,
+)
+from litellm.litellm_core_utils.prompt_templates.common_utils import get_tool_call_names
+from litellm.litellm_core_utils.prompt_templates.image_handling import (
+ async_convert_url_to_base64,
+ convert_url_to_base64,
+)
+from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
+from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
+from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import (
+ AllMessageValues,
+ ChatCompletionFileObject,
+ ChatCompletionFileObjectFile,
+ ChatCompletionImageObject,
+ ChatCompletionImageUrlObject,
+ OpenAIChatCompletionChoices,
+ OpenAIMessageContentListBlock,
+)
+from litellm.types.utils import (
+ ChatCompletionMessageToolCall,
+ Choices,
+ Function,
+ Message,
+ ModelResponse,
+ ModelResponseStream,
+)
+from litellm.utils import convert_to_model_response_object
+
+from ..common_utils import OpenAIError
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+ from litellm.llms.base_llm.base_utils import BaseTokenCounter
+ from litellm.types.llms.openai import ChatCompletionToolParam
+
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+ LiteLLMLoggingObj = Any
+
+
+class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
+ """
+ Reference: https://platform.openai.com/docs/api-reference/chat/create
+
+ The class `OpenAIConfig` provides configuration for the OpenAI's Chat API interface. Below are the parameters:
+
+ - `frequency_penalty` (number or null): Defaults to 0. Allows a value between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, thereby minimizing repetition.
+
+ - `function_call` (string or object): This optional parameter controls how the model calls functions.
+
+ - `functions` (array): An optional parameter. It is a list of functions for which the model may generate JSON inputs.
+
+ - `logit_bias` (map): This optional parameter modifies the likelihood of specified tokens appearing in the completion.
+
+ - `max_tokens` (integer or null): This optional parameter helps to set the maximum number of tokens to generate in the chat completion.
+
+ - `n` (integer or null): This optional parameter helps to set how many chat completion choices to generate for each input message.
+
+ - `presence_penalty` (number or null): Defaults to 0. It penalizes new tokens based on if they appear in the text so far, hence increasing the model's likelihood to talk about new topics.
+
+ - `stop` (string / array / null): Specifies up to 4 sequences where the API will stop generating further tokens.
+
+ - `temperature` (number or null): Defines the sampling temperature to use, varying between 0 and 2.
+
+ - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling.
+ """
+
+ # Add a class variable to track if this is the base class
+ _is_base_class = True
+
+ frequency_penalty: Optional[int] = None
+ function_call: Optional[Union[str, dict]] = None
+ functions: Optional[list] = None
+ logit_bias: Optional[dict] = None
+ max_tokens: Optional[int] = None
+ n: Optional[int] = None
+ presence_penalty: Optional[int] = None
+ stop: Optional[Union[str, list]] = None
+ temperature: Optional[int] = None
+ top_p: Optional[int] = None
+ response_format: Optional[dict] = None
+
+ def __init__(
+ self,
+ frequency_penalty: Optional[int] = None,
+ function_call: Optional[Union[str, dict]] = None,
+ functions: Optional[list] = None,
+ logit_bias: Optional[dict] = None,
+ max_tokens: Optional[int] = None,
+ n: Optional[int] = None,
+ presence_penalty: Optional[int] = None,
+ stop: Optional[Union[str, list]] = None,
+ temperature: Optional[int] = None,
+ top_p: Optional[int] = None,
+ response_format: Optional[dict] = None,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ self.__class__._is_base_class = False
+
+ @classmethod
+ def get_config(cls):
+ return super().get_config()
+
+ def get_supported_openai_params(self, model: str) -> list:
+ base_params = [
+ "frequency_penalty",
+ "logit_bias",
+ "logprobs",
+ "top_logprobs",
+ "max_tokens",
+ "max_completion_tokens",
+ "modalities",
+ "prediction",
+ "n",
+ "presence_penalty",
+ "seed",
+ "stop",
+ "stream",
+ "stream_options",
+ "temperature",
+ "top_p",
+ "tools",
+ "tool_choice",
+ "function_call",
+ "functions",
+ "max_retries",
+ "extra_headers",
+ "parallel_tool_calls",
+ "audio",
+ "web_search_options",
+ "service_tier",
+ "safety_identifier",
+ "prompt_cache_key",
+ "prompt_cache_retention",
+ "store",
+ ] # works across all models
+
+ model_specific_params = []
+ if (
+ model != "gpt-3.5-turbo-16k" and model != "gpt-4"
+ ): # gpt-4 does not support 'response_format'
+ model_specific_params.append("response_format")
+
+ # Normalize model name for responses API (e.g., "responses/gpt-4.1" -> "gpt-4.1")
+ model_for_check = (
+ model.split("responses/", 1)[1] if "responses/" in model else model
+ )
+ if (
+ model_for_check in litellm.open_ai_chat_completion_models
+ ) or model_for_check in litellm.open_ai_text_completion_models:
+ model_specific_params.append(
+ "user"
+ ) # user is not a param supported by all openai-compatible endpoints - e.g. azure ai
+ return base_params + model_specific_params
+
+ def _map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ """
+ If any supported_openai_params are in non_default_params, add them to optional_params, so they are use in API call
+
+ Args:
+ non_default_params (dict): Non-default parameters to filter.
+ optional_params (dict): Optional parameters to update.
+ model (str): Model name for parameter support check.
+
+ Returns:
+ dict: Updated optional_params with supported non-default parameters.
+ """
+ supported_openai_params = self.get_supported_openai_params(model)
+ for param, value in non_default_params.items():
+ if param in supported_openai_params:
+ optional_params[param] = value
+ return optional_params
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ return self._map_openai_params(
+ non_default_params=non_default_params,
+ optional_params=optional_params,
+ model=model,
+ drop_params=drop_params,
+ )
+
+ def contains_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> bool:
+ potential_pdf_url_starts = ["https://", "http://", "www."]
+ file_id = content_item.get("file_id")
+ if file_id and any(
+ file_id.startswith(start) for start in potential_pdf_url_starts
+ ):
+ return True
+ return False
+
+ def _handle_pdf_url(
+ self, content_item: ChatCompletionFileObjectFile
+ ) -> ChatCompletionFileObjectFile:
+ content_copy = content_item.copy()
+ file_id = content_copy.get("file_id")
+ if file_id is not None:
+ base64_data = convert_url_to_base64(file_id)
+ content_copy["file_data"] = base64_data
+ content_copy["filename"] = "my_file.pdf"
+ content_copy.pop("file_id")
+ return content_copy
+
+ async def _async_handle_pdf_url(
+ self, content_item: ChatCompletionFileObjectFile
+ ) -> ChatCompletionFileObjectFile:
+ file_id = content_item.get("file_id")
+ if file_id is not None: # check for file id being url done in _handle_pdf_url
+ base64_data = await async_convert_url_to_base64(file_id)
+ content_item["file_data"] = base64_data
+ content_item["filename"] = "my_file.pdf"
+ content_item.pop("file_id")
+ return content_item
+
+ def _common_file_data_check(
+ self, content_item: ChatCompletionFileObjectFile
+ ) -> ChatCompletionFileObjectFile:
+ file_data = content_item.get("file_data")
+ filename = content_item.get("filename")
+ if file_data is not None and filename is None:
+ content_item["filename"] = "my_file.pdf"
+ return content_item
+
+ def _apply_common_transform_content_item(
+ self,
+ content_item: OpenAIMessageContentListBlock,
+ ) -> OpenAIMessageContentListBlock:
+ litellm_specific_params = {"format"}
+ if content_item.get("type") == "image_url":
+ content_item = cast(ChatCompletionImageObject, content_item)
+ if isinstance(content_item["image_url"], str):
+ content_item["image_url"] = {
+ "url": content_item["image_url"],
+ }
+ elif isinstance(content_item["image_url"], dict):
+ new_image_url_obj = ChatCompletionImageUrlObject(
+ **{ # type: ignore
+ k: v
+ for k, v in content_item["image_url"].items()
+ if k not in litellm_specific_params
+ }
+ )
+ content_item["image_url"] = new_image_url_obj
+ elif content_item.get("type") == "file":
+ content_item = cast(ChatCompletionFileObject, content_item)
+ file_obj = content_item["file"]
+ new_file_obj = ChatCompletionFileObjectFile(
+ **{ # type: ignore
+ k: v
+ for k, v in file_obj.items()
+ if k not in litellm_specific_params
+ }
+ )
+ content_item["file"] = new_file_obj
+
+ return content_item
+
+ def _transform_content_item(
+ self,
+ content_item: OpenAIMessageContentListBlock,
+ ) -> OpenAIMessageContentListBlock:
+ content_item = self._apply_common_transform_content_item(content_item)
+ content_item_type = content_item.get("type")
+ potential_file_obj = content_item.get("file")
+ if content_item_type == "file" and potential_file_obj:
+ file_obj = cast(ChatCompletionFileObjectFile, potential_file_obj)
+ content_item_typed = cast(ChatCompletionFileObject, content_item)
+ if self.contains_pdf_url(file_obj):
+ file_obj = self._handle_pdf_url(file_obj)
+ file_obj = self._common_file_data_check(file_obj)
+ content_item_typed["file"] = file_obj
+ content_item = content_item_typed
+ return content_item
+
+ async def _async_transform_content_item(
+ self, content_item: OpenAIMessageContentListBlock, is_async: bool = False
+ ) -> OpenAIMessageContentListBlock:
+ content_item = self._apply_common_transform_content_item(content_item)
+ content_item_type = content_item.get("type")
+ potential_file_obj = content_item.get("file")
+ if content_item_type == "file" and potential_file_obj:
+ file_obj = cast(ChatCompletionFileObjectFile, potential_file_obj)
+ content_item_typed = cast(ChatCompletionFileObject, content_item)
+ if self.contains_pdf_url(file_obj):
+ file_obj = await self._async_handle_pdf_url(file_obj)
+ file_obj = self._common_file_data_check(file_obj)
+ content_item_typed["file"] = file_obj
+ content_item = content_item_typed
+ return content_item
+
+ # fmt: off
+
+ @overload
+ def _transform_messages(
+ self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
+ ) -> Coroutine[Any, Any, List[AllMessageValues]]:
+ ...
+
+ @overload
+ def _transform_messages(
+ self,
+ messages: List[AllMessageValues],
+ model: str,
+ is_async: Literal[False] = False,
+ ) -> List[AllMessageValues]:
+ ...
+
+ # fmt: on
+
+ def _transform_messages(
+ self, messages: List[AllMessageValues], model: str, is_async: bool = False
+ ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
+ """OpenAI no longer supports image_url as a string, so we need to convert it to a dict"""
+
+ async def _async_transform():
+ for message in messages:
+ message_content = message.get("content")
+ message_role = message.get("role")
+
+ if (
+ message_role == "user"
+ and message_content
+ and isinstance(message_content, list)
+ ):
+ message_content_types = cast(
+ List[OpenAIMessageContentListBlock], message_content
+ )
+ for i, content_item in enumerate(message_content_types):
+ message_content_types[
+ i
+ ] = await self._async_transform_content_item(
+ cast(OpenAIMessageContentListBlock, content_item),
+ )
+ return messages
+
+ if is_async:
+ return _async_transform()
+ else:
+ for message in messages:
+ message_content = message.get("content")
+ message_role = message.get("role")
+ if (
+ message_role == "user"
+ and message_content
+ and isinstance(message_content, list)
+ ):
+ message_content_types = cast(
+ List[OpenAIMessageContentListBlock], message_content
+ )
+ for i, content_item in enumerate(message_content):
+ message_content_types[i] = self._transform_content_item(
+ cast(OpenAIMessageContentListBlock, content_item)
+ )
+ return messages
+
+ def remove_cache_control_flag_from_messages_and_tools(
+ self,
+ model: str, # allows overrides to selectively run this
+ messages: List[AllMessageValues],
+ tools: Optional[List["ChatCompletionToolParam"]] = None,
+ ) -> Tuple[List[AllMessageValues], Optional[List["ChatCompletionToolParam"]]]:
+ from litellm.litellm_core_utils.prompt_templates.common_utils import (
+ filter_value_from_dict,
+ )
+ from litellm.types.llms.openai import ChatCompletionToolParam
+
+ for i, message in enumerate(messages):
+ messages[i] = cast(
+ AllMessageValues, filter_value_from_dict(message, "cache_control") # type: ignore
+ )
+ if tools is not None:
+ for i, tool in enumerate(tools):
+ tools[i] = cast(
+ ChatCompletionToolParam,
+ filter_value_from_dict(tool, "cache_control"), # type: ignore
+ )
+ return messages, tools
+
+ def transform_request(
+ self,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ headers: dict,
+ ) -> dict:
+ """
+ Transform the overall request to be sent to the API.
+
+ Returns:
+ dict: The transformed request. Sent as the body of the API call.
+ """
+ messages = self._transform_messages(messages=messages, model=model)
+ messages, tools = self.remove_cache_control_flag_from_messages_and_tools(
+ model=model, messages=messages, tools=optional_params.get("tools", [])
+ )
+ if tools is not None and len(tools) > 0:
+ optional_params["tools"] = tools
+
+ optional_params.pop("max_retries", None)
+
+ return {
+ "model": model,
+ "messages": messages,
+ **optional_params,
+ }
+
+ async def async_transform_request(
+ self,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ headers: dict,
+ ) -> dict:
+ transformed_messages = await self._transform_messages(
+ messages=messages, model=model, is_async=True
+ )
+ (
+ transformed_messages,
+ tools,
+ ) = self.remove_cache_control_flag_from_messages_and_tools(
+ model=model,
+ messages=transformed_messages,
+ tools=optional_params.get("tools", []),
+ )
+ if tools is not None and len(tools) > 0:
+ optional_params["tools"] = tools
+ if self.__class__._is_base_class:
+ return {
+ "model": model,
+ "messages": transformed_messages,
+ **optional_params,
+ }
+ else:
+ ## allow for any object specific behaviour to be handled
+ return self.transform_request(
+ model, messages, optional_params, litellm_params, headers
+ )
+
+ def _passed_in_tools(self, optional_params: dict) -> bool:
+ return optional_params.get("tools", None) is not None
+
+ def _check_and_fix_if_content_is_tool_call(
+ self, content: str, optional_params: dict
+ ) -> Optional[ChatCompletionMessageToolCall]:
+ """
+ Check if the content is a tool call
+ """
+ import json
+
+ if not self._passed_in_tools(optional_params):
+ return None
+ tool_call_names = get_tool_call_names(optional_params.get("tools", []))
+ try:
+ json_content = json.loads(content)
+ if (
+ json_content.get("type") == "function"
+ and json_content.get("name") in tool_call_names
+ ):
+ return ChatCompletionMessageToolCall(
+ function=Function(
+ name=json_content.get("name"),
+ arguments=json_content.get("arguments"),
+ )
+ )
+ except Exception:
+ return None
+
+ return None
+
+ def _get_finish_reason(self, message: Message, received_finish_reason: str) -> str:
+ if message.tool_calls is not None:
+ return "tool_calls"
+ else:
+ return received_finish_reason
+
+ def _transform_choices(
+ self,
+ choices: List[OpenAIChatCompletionChoices],
+ json_mode: Optional[bool] = None,
+ optional_params: Optional[dict] = None,
+ ) -> List[Choices]:
+ transformed_choices = []
+
+ for choice in choices:
+ ## HANDLE JSON MODE - anthropic returns single function call]
+ tool_calls = choice["message"].get("tool_calls", None)
+ new_tool_calls: Optional[List[ChatCompletionMessageToolCall]] = None
+ message_content = choice["message"].get("content", None)
+ if tool_calls is not None:
+ _openai_tool_calls = []
+ for _tc in tool_calls:
+ _openai_tc = ChatCompletionMessageToolCall(**_tc) # type: ignore
+ _openai_tool_calls.append(_openai_tc)
+ fixed_tool_calls = _handle_invalid_parallel_tool_calls(
+ _openai_tool_calls
+ )
+
+ if fixed_tool_calls is not None:
+ new_tool_calls = fixed_tool_calls
+ elif (
+ optional_params is not None
+ and message_content
+ and isinstance(message_content, str)
+ ):
+ new_tool_call = self._check_and_fix_if_content_is_tool_call(
+ message_content, optional_params
+ )
+ if new_tool_call is not None:
+ choice["message"]["content"] = None # remove the content
+ new_tool_calls = [new_tool_call]
+
+ translated_message: Optional[Message] = None
+ finish_reason: Optional[str] = None
+ if new_tool_calls and _should_convert_tool_call_to_json_mode(
+ tool_calls=new_tool_calls,
+ convert_tool_call_to_json_mode=json_mode,
+ ):
+ # to support response_format on claude models
+ json_mode_content_str: Optional[str] = (
+ str(new_tool_calls[0]["function"].get("arguments", "")) or None
+ )
+ if json_mode_content_str is not None:
+ translated_message = Message(content=json_mode_content_str)
+ finish_reason = "stop"
+
+ if translated_message is None:
+ ## get the reasoning content
+ (
+ reasoning_content,
+ content_str,
+ ) = _extract_reasoning_content(cast(dict, choice["message"]))
+
+ translated_message = Message(
+ role="assistant",
+ content=content_str,
+ reasoning_content=reasoning_content,
+ thinking_blocks=None,
+ tool_calls=new_tool_calls,
+ )
+
+ if finish_reason is None:
+ finish_reason = choice["finish_reason"]
+
+ translated_choice = Choices(
+ finish_reason=finish_reason,
+ index=choice["index"],
+ message=translated_message,
+ logprobs=None,
+ enhancements=None,
+ )
+
+ translated_choice.finish_reason = map_finish_reason(
+ self._get_finish_reason(translated_message, choice["finish_reason"])
+ )
+ transformed_choices.append(translated_choice)
+
+ return transformed_choices
+
+ def transform_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ model_response: ModelResponse,
+ logging_obj: LiteLLMLoggingObj,
+ request_data: dict,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: Any,
+ api_key: Optional[str] = None,
+ json_mode: Optional[bool] = None,
+ ) -> ModelResponse:
+ """
+ Transform the response from the API.
+
+ Returns:
+ dict: The transformed response.
+ """
+
+ ## LOGGING
+ logging_obj.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=raw_response.text,
+ additional_args={"complete_input_dict": request_data},
+ )
+
+ ## RESPONSE OBJECT
+ try:
+ completion_response = raw_response.json()
+ except Exception as e:
+ response_headers = getattr(raw_response, "headers", None)
+ raise OpenAIError(
+ message="Unable to get json response - {}, Original Response: {}".format(
+ str(e), raw_response.text
+ ),
+ status_code=raw_response.status_code,
+ headers=response_headers,
+ )
+ raw_response_headers = dict(raw_response.headers)
+ final_response_obj = convert_to_model_response_object(
+ response_object=completion_response,
+ model_response_object=model_response,
+ hidden_params={"headers": raw_response_headers},
+ _response_headers=raw_response_headers,
+ )
+
+ return cast(ModelResponse, final_response_obj)
+
+ def get_error_class(
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+ ) -> BaseLLMException:
+ return OpenAIError(
+ status_code=status_code,
+ message=error_message,
+ headers=cast(httpx.Headers, headers),
+ )
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ api_key: Optional[str],
+ model: str,
+ optional_params: dict,
+ litellm_params: dict,
+ stream: Optional[bool] = None,
+ ) -> str:
+ """
+ Get the complete URL for the API call.
+
+ Returns:
+ str: The complete URL for the API call.
+ """
+ if api_base is None:
+ api_base = "https://api.openai.com"
+ endpoint = "chat/completions"
+
+ # Remove trailing slash from api_base if present
+ api_base = api_base.rstrip("/")
+
+ # Check if endpoint is already in the api_base
+ if endpoint in api_base:
+ return api_base
+
+ return f"{api_base}/{endpoint}"
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ if api_key is not None:
+ headers["Authorization"] = f"Bearer {api_key}"
+
+ # Ensure Content-Type is set to application/json
+ if "content-type" not in headers and "Content-Type" not in headers:
+ headers["Content-Type"] = "application/json"
+
+ return headers
+
+ def get_models(
+ self, api_key: Optional[str] = None, api_base: Optional[str] = None
+ ) -> List[str]:
+ """
+ Calls OpenAI's `/v1/models` endpoint and returns the list of models.
+ """
+
+ if api_base is None:
+ api_base = "https://api.openai.com"
+ if api_key is None:
+ api_key = get_secret_str("OPENAI_API_KEY")
+
+ # Strip api_base to just the base URL (scheme + host + port)
+ parsed_url = httpx.URL(api_base)
+ base_url = f"{parsed_url.scheme}://{parsed_url.host}"
+ if parsed_url.port:
+ base_url += f":{parsed_url.port}"
+
+ response = litellm.module_level_client.get(
+ url=f"{base_url}/v1/models",
+ headers={"Authorization": f"Bearer {api_key}"},
+ )
+
+ if response.status_code != 200:
+ raise Exception(f"Failed to get models: {response.text}")
+
+ models = response.json()["data"]
+ return [model["id"] for model in models]
+
+ @staticmethod
+ def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
+ return (
+ api_key
+ or litellm.api_key
+ or litellm.openai_key
+ or get_secret_str("OPENAI_API_KEY")
+ )
+
+ @staticmethod
+ def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
+ return (
+ api_base
+ or litellm.api_base
+ or get_secret_str("OPENAI_BASE_URL")
+ or get_secret_str("OPENAI_API_BASE")
+ or "https://api.openai.com/v1"
+ )
+
+ @staticmethod
+ def get_base_model(model: Optional[str] = None) -> Optional[str]:
+ return model
+
+ def get_token_counter(self) -> Optional["BaseTokenCounter"]:
+ from litellm.llms.openai.responses.count_tokens.token_counter import (
+ OpenAITokenCounter,
+ )
+
+ return OpenAITokenCounter()
+
+ def get_model_response_iterator(
+ self,
+ streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
+ sync_stream: bool,
+ json_mode: Optional[bool] = False,
+ ) -> Any:
+ return OpenAIChatCompletionStreamingHandler(
+ streaming_response=streaming_response,
+ sync_stream=sync_stream,
+ json_mode=json_mode,
+ )
+
+
+class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
+ def _map_reasoning_to_reasoning_content(self, choices: list) -> list:
+ """
+ Map 'reasoning' field to 'reasoning_content' field in delta.
+
+ Some OpenAI-compatible providers (e.g., GLM-5, hosted_vllm) return
+ delta.reasoning, but LiteLLM expects delta.reasoning_content.
+
+ Args:
+ choices: List of choice objects from the streaming chunk
+
+ Returns:
+ List of choices with reasoning field mapped to reasoning_content
+ """
+ for choice in choices:
+ delta = choice.get("delta", {})
+ if "reasoning" in delta:
+ delta["reasoning_content"] = delta.pop("reasoning")
+ return choices
+
+ def chunk_parser(self, chunk: dict) -> ModelResponseStream:
+ try:
+ choices = chunk.get("choices", [])
+ choices = self._map_reasoning_to_reasoning_content(choices)
+
+ kwargs: Dict[str, Any] = {
+ "id": chunk.get("id"),
+ "object": "chat.completion.chunk",
+ "created": chunk.get("created"),
+ "model": chunk.get("model"),
+ "choices": choices,
+ }
+ if "usage" in chunk and chunk["usage"] is not None:
+ kwargs["usage"] = chunk["usage"]
+ return ModelResponseStream(**kwargs)
+ except Exception as e:
+ raise e
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/guardrail_translation/README.md b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/guardrail_translation/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..05e3b55e54c8d786b34a51f562ca7e2acbbdfa0a
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/guardrail_translation/README.md
@@ -0,0 +1,3 @@
+Translation of OpenAI `/chat/completions` input and output to a custom guardrail.
+
+This enables guardrails to be applied to OpenAI `/chat/completions` requests and responses.
\ No newline at end of file
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/guardrail_translation/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/guardrail_translation/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0682aa47588577ff1ba561889bbe4ff7ac0b6be
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/guardrail_translation/__init__.py
@@ -0,0 +1,12 @@
+"""OpenAI Chat Completions message handler for Unified Guardrails."""
+
+from litellm.llms.openai.chat.guardrail_translation.handler import (
+ OpenAIChatCompletionsHandler,
+)
+from litellm.types.utils import CallTypes
+
+guardrail_translation_mappings = {
+ CallTypes.completion: OpenAIChatCompletionsHandler,
+ CallTypes.acompletion: OpenAIChatCompletionsHandler,
+}
+__all__ = ["guardrail_translation_mappings"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/guardrail_translation/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/guardrail_translation/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..bab4c3b5eb7516c467f87afc4985f881e5559409
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/guardrail_translation/handler.py
@@ -0,0 +1,808 @@
+"""
+OpenAI Chat Completions Message Handler for Unified Guardrails
+
+This module provides a class-based handler for OpenAI-format chat completions.
+The class methods can be overridden for custom behavior.
+
+Pattern Overview:
+-----------------
+1. Extract text content from messages/responses (both string and list formats)
+2. Create async tasks to apply guardrails to each text segment
+3. Track mappings to know where each response belongs
+4. Apply guardrail responses back to the original structure
+
+This pattern can be replicated for other message formats (e.g., Anthropic).
+"""
+
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
+
+import litellm
+from litellm._logging import verbose_proxy_logger
+from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
+from litellm.main import stream_chunk_builder
+from litellm.types.llms.openai import ChatCompletionToolParam
+from litellm.types.utils import (
+ Choices,
+ GenericGuardrailAPIInputs,
+ ModelResponse,
+ ModelResponseStream,
+ StreamingChoices,
+)
+
+if TYPE_CHECKING:
+ from litellm.integrations.custom_guardrail import CustomGuardrail
+
+
+class OpenAIChatCompletionsHandler(BaseTranslation):
+ """
+ Handler for processing OpenAI chat completions messages with guardrails.
+
+ This class provides methods to:
+ 1. Process input messages (pre-call hook)
+ 2. Process output responses (post-call hook)
+
+ Methods can be overridden to customize behavior for different message formats.
+ """
+
+ async def process_input_messages(
+ self,
+ data: dict,
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process input messages by applying guardrails to text content.
+ """
+ messages = data.get("messages")
+ if messages is None:
+ return data
+
+ texts_to_check: List[str] = []
+ images_to_check: List[str] = []
+ tool_calls_to_check: List[ChatCompletionToolParam] = []
+ text_task_mappings: List[Tuple[int, Optional[int]]] = []
+ tool_call_task_mappings: List[Tuple[int, int]] = []
+ # text_task_mappings: Track (message_index, content_index) for each text
+ # content_index is None for string content, int for list content
+ # tool_call_task_mappings: Track (message_index, tool_call_index) for each tool call
+
+ # Step 1: Extract all text content, images, and tool calls
+ for msg_idx, message in enumerate(messages):
+ self._extract_inputs(
+ message=message,
+ msg_idx=msg_idx,
+ texts_to_check=texts_to_check,
+ images_to_check=images_to_check,
+ tool_calls_to_check=tool_calls_to_check,
+ text_task_mappings=text_task_mappings,
+ tool_call_task_mappings=tool_call_task_mappings,
+ )
+
+ # Step 2: Apply guardrail to all texts and tool calls in batch
+ if texts_to_check or tool_calls_to_check:
+ inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
+ if images_to_check:
+ inputs["images"] = images_to_check
+ if tool_calls_to_check:
+ inputs["tool_calls"] = tool_calls_to_check # type: ignore
+ if messages:
+ inputs[
+ "structured_messages"
+ ] = messages # pass the openai /chat/completions messages to the guardrail, as-is
+ # Pass tools (function definitions) to the guardrail
+ tools = data.get("tools")
+ if tools:
+ inputs["tools"] = tools
+ # Include model information if available
+ model = data.get("model")
+ if model:
+ inputs["model"] = model
+
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=data,
+ input_type="request",
+ logging_obj=litellm_logging_obj,
+ )
+
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+ guardrailed_tool_calls = guardrailed_inputs.get("tool_calls", [])
+ guardrailed_tools = guardrailed_inputs.get("tools")
+ if guardrailed_tools is not None:
+ data["tools"] = guardrailed_tools
+
+ # Step 3: Map guardrail responses back to original message structure
+ if guardrailed_texts and texts_to_check:
+ await self._apply_guardrail_responses_to_input_texts(
+ messages=messages,
+ responses=guardrailed_texts,
+ task_mappings=text_task_mappings,
+ )
+
+ # Step 4: Apply guardrailed tool calls back to messages
+ if guardrailed_tool_calls:
+ # Note: The guardrail may modify tool_calls_to_check in place
+ # or we may need to handle returned tool calls differently
+ await self._apply_guardrail_responses_to_input_tool_calls(
+ messages=messages,
+ tool_calls=guardrailed_tool_calls, # type: ignore
+ task_mappings=tool_call_task_mappings,
+ )
+
+ verbose_proxy_logger.debug(
+ "OpenAI Chat Completions: Processed input messages: %s", messages
+ )
+
+ return data
+
+ def extract_request_tool_names(self, data: dict) -> List[str]:
+ """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name)."""
+ names: List[str] = []
+ for tool in data.get("tools") or []:
+ if isinstance(tool, dict) and tool.get("type") == "function":
+ fn = tool.get("function")
+ if isinstance(fn, dict) and fn.get("name"):
+ names.append(str(fn["name"]))
+ for fn in data.get("functions") or []:
+ if isinstance(fn, dict) and fn.get("name"):
+ names.append(str(fn["name"]))
+ return names
+
+ def _extract_inputs(
+ self,
+ message: Dict[str, Any],
+ msg_idx: int,
+ texts_to_check: List[str],
+ images_to_check: List[str],
+ tool_calls_to_check: List[ChatCompletionToolParam],
+ text_task_mappings: List[Tuple[int, Optional[int]]],
+ tool_call_task_mappings: List[Tuple[int, int]],
+ ) -> None:
+ """
+ Extract text content, images, and tool calls from a message.
+
+ Override this method to customize text/image/tool call extraction logic.
+ """
+ content = message.get("content", None)
+ if content is not None:
+ if isinstance(content, str):
+ # Simple string content
+ texts_to_check.append(content)
+ text_task_mappings.append((msg_idx, None))
+
+ elif isinstance(content, list):
+ # List content (e.g., multimodal with text and images)
+ for content_idx, content_item in enumerate(content):
+ # Extract text
+ text_str = content_item.get("text", None)
+ if text_str is not None:
+ texts_to_check.append(text_str)
+ text_task_mappings.append((msg_idx, int(content_idx)))
+
+ # Extract images (image_url)
+ if content_item.get("type") == "image_url":
+ image_url = content_item.get("image_url", {})
+ if isinstance(image_url, dict):
+ url = image_url.get("url")
+ if url:
+ images_to_check.append(url)
+ elif isinstance(image_url, str):
+ images_to_check.append(image_url)
+
+ # Extract tool calls (typically in assistant messages)
+ tool_calls = message.get("tool_calls", None)
+ if tool_calls is not None and isinstance(tool_calls, list):
+ for tool_call_idx, tool_call in enumerate(tool_calls):
+ if isinstance(tool_call, dict):
+ # Add the full tool call object to the list
+ tool_calls_to_check.append(cast(ChatCompletionToolParam, tool_call))
+ tool_call_task_mappings.append((msg_idx, int(tool_call_idx)))
+
+ async def _apply_guardrail_responses_to_input_texts(
+ self,
+ messages: List[Dict[str, Any]],
+ responses: List[str],
+ task_mappings: List[Tuple[int, Optional[int]]],
+ ) -> None:
+ """
+ Apply guardrail responses back to input message text content.
+
+ Override this method to customize how text responses are applied.
+ """
+ for task_idx, guardrail_response in enumerate(responses):
+ mapping = task_mappings[task_idx]
+ msg_idx = cast(int, mapping[0])
+ content_idx_optional = cast(Optional[int], mapping[1])
+
+ # Handle content
+ content = messages[msg_idx].get("content", None)
+ if content is None:
+ continue
+
+ if isinstance(content, str) and content_idx_optional is None:
+ # Replace string content with guardrail response
+ messages[msg_idx]["content"] = guardrail_response
+
+ elif isinstance(content, list) and content_idx_optional is not None:
+ # Replace specific text item in list content
+ messages[msg_idx]["content"][content_idx_optional][
+ "text"
+ ] = guardrail_response
+
+ async def _apply_guardrail_responses_to_input_tool_calls(
+ self,
+ messages: List[Dict[str, Any]],
+ tool_calls: List[Dict[str, Any]],
+ task_mappings: List[Tuple[int, int]],
+ ) -> None:
+ """
+ Apply guardrailed tool calls back to input messages.
+
+ The guardrail may have modified the tool_calls list in place,
+ so we apply the modified tool calls back to the original messages.
+
+ Override this method to customize how tool call responses are applied.
+ """
+ for task_idx, (msg_idx, tool_call_idx) in enumerate(task_mappings):
+ if task_idx < len(tool_calls):
+ guardrailed_tool_call = tool_calls[task_idx]
+ message_tool_calls = messages[msg_idx].get("tool_calls", None)
+ if message_tool_calls is not None and isinstance(
+ message_tool_calls, list
+ ):
+ if tool_call_idx < len(message_tool_calls):
+ # Replace the tool call with the guardrailed version
+ message_tool_calls[tool_call_idx] = guardrailed_tool_call
+
+ async def process_output_response(
+ self,
+ response: "ModelResponse",
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ user_api_key_dict: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process output response by applying guardrails to text content.
+
+ Args:
+ response: LiteLLM ModelResponse object
+ guardrail_to_apply: The guardrail instance to apply
+ litellm_logging_obj: Optional logging object
+ user_api_key_dict: User API key metadata to pass to guardrails
+
+ Returns:
+ Modified response with guardrail applied to content
+
+ Response Format Support:
+ - String content: choice.message.content = "text here"
+ - List content: choice.message.content = [{"type": "text", "text": "text here"}, ...]
+ """
+
+ # Step 0: Check if response has any text content to process
+ if not self._has_text_content(response):
+ verbose_proxy_logger.warning(
+ "OpenAI Chat Completions: No text content in response, skipping guardrail"
+ )
+ return response
+
+ texts_to_check: List[str] = []
+ images_to_check: List[str] = []
+ tool_calls_to_check: List[Dict[str, Any]] = []
+ text_task_mappings: List[Tuple[int, Optional[int]]] = []
+ tool_call_task_mappings: List[Tuple[int, int]] = []
+ # text_task_mappings: Track (choice_index, content_index) for each text
+ # content_index is None for string content, int for list content
+ # tool_call_task_mappings: Track (choice_index, tool_call_index) for each tool call
+
+ # Step 1: Extract all text content, images, and tool calls from response choices
+ for choice_idx, choice in enumerate(response.choices):
+ self._extract_output_text_images_and_tool_calls(
+ choice=choice,
+ choice_idx=choice_idx,
+ texts_to_check=texts_to_check,
+ images_to_check=images_to_check,
+ tool_calls_to_check=tool_calls_to_check,
+ text_task_mappings=text_task_mappings,
+ tool_call_task_mappings=tool_call_task_mappings,
+ )
+
+ # Step 2: Apply guardrail to all texts and tool calls in batch
+ if texts_to_check or tool_calls_to_check:
+ # Create a request_data dict with response info and user API key metadata
+ request_data: dict = {"response": response}
+
+ # Add user API key metadata with prefixed keys
+ user_metadata = self.transform_user_api_key_dict_to_metadata(
+ user_api_key_dict
+ )
+ if user_metadata:
+ request_data["litellm_metadata"] = user_metadata
+
+ inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
+ if images_to_check:
+ inputs["images"] = images_to_check
+ if tool_calls_to_check:
+ inputs["tool_calls"] = tool_calls_to_check # type: ignore
+ # Include model information from the response if available
+ if hasattr(response, "model") and response.model:
+ inputs["model"] = response.model
+
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=request_data,
+ input_type="response",
+ logging_obj=litellm_logging_obj,
+ )
+
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+
+ # Step 3: Map guardrail responses back to original response structure
+ if guardrailed_texts and texts_to_check:
+ await self._apply_guardrail_responses_to_output_texts(
+ response=response,
+ responses=guardrailed_texts,
+ task_mappings=text_task_mappings,
+ )
+
+ # Step 4: Apply guardrailed tool calls back to response
+ if tool_calls_to_check:
+ await self._apply_guardrail_responses_to_output_tool_calls(
+ response=response,
+ tool_calls=tool_calls_to_check,
+ task_mappings=tool_call_task_mappings,
+ )
+
+ verbose_proxy_logger.debug(
+ "OpenAI Chat Completions: Processed output response: %s", response
+ )
+
+ return response
+
+ async def process_output_streaming_response(
+ self,
+ responses_so_far: List["ModelResponseStream"],
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ user_api_key_dict: Optional[Any] = None,
+ ) -> List["ModelResponseStream"]:
+ """
+ Process output streaming responses by applying guardrails to text content.
+
+ Args:
+ responses_so_far: List of LiteLLM ModelResponseStream objects
+ guardrail_to_apply: The guardrail instance to apply
+ litellm_logging_obj: Optional logging object
+ user_api_key_dict: User API key metadata to pass to guardrails
+
+ Returns:
+ Modified list of responses with guardrail applied to content
+
+ Response Format Support:
+ - String content: choice.message.content = "text here"
+ - List content: choice.message.content = [{"type": "text", "text": "text here"}, ...]
+ """
+ # check if the stream has ended
+ has_stream_ended = False
+ for chunk in responses_so_far:
+ if chunk.choices and chunk.choices[0].finish_reason is not None:
+ has_stream_ended = True
+ break
+
+ if has_stream_ended:
+ # convert to model response
+ model_response = cast(
+ ModelResponse,
+ stream_chunk_builder(
+ chunks=responses_so_far, logging_obj=litellm_logging_obj
+ ),
+ )
+ # run process_output_response
+ await self.process_output_response(
+ response=model_response,
+ guardrail_to_apply=guardrail_to_apply,
+ litellm_logging_obj=litellm_logging_obj,
+ user_api_key_dict=user_api_key_dict,
+ )
+
+ return responses_so_far
+
+ # Step 0: Check if any response has text content to process
+ has_any_text_content = False
+ for response in responses_so_far:
+ if self._has_text_content(response):
+ has_any_text_content = True
+ break
+
+ if not has_any_text_content:
+ verbose_proxy_logger.warning(
+ "OpenAI Chat Completions: No text content in streaming responses, skipping guardrail"
+ )
+ return responses_so_far
+
+ # Step 1: Combine all streaming chunks into complete text per choice
+ # For streaming, we need to concatenate all delta.content across all chunks
+ # Key: (choice_idx, content_idx), Value: combined text
+ combined_texts = self._combine_streaming_texts(responses_so_far)
+
+ # Step 2: Create lists for guardrail processing
+ texts_to_check: List[str] = []
+ images_to_check: List[str] = []
+ task_mappings: List[Tuple[int, Optional[int]]] = []
+ # Track (choice_index, content_index) for each combined text
+
+ for (map_choice_idx, map_content_idx), combined_text in combined_texts.items():
+ texts_to_check.append(combined_text)
+ task_mappings.append((map_choice_idx, map_content_idx))
+
+ # Step 3: Apply guardrail to all combined texts in batch
+ if texts_to_check:
+ # Create a request_data dict with response info and user API key metadata
+ request_data: dict = {"responses": responses_so_far}
+
+ # Add user API key metadata with prefixed keys
+ user_metadata = self.transform_user_api_key_dict_to_metadata(
+ user_api_key_dict
+ )
+ if user_metadata:
+ request_data["litellm_metadata"] = user_metadata
+
+ inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
+ if images_to_check:
+ inputs["images"] = images_to_check
+ # Include model information from the first response if available
+ if (
+ responses_so_far
+ and hasattr(responses_so_far[0], "model")
+ and responses_so_far[0].model
+ ):
+ inputs["model"] = responses_so_far[0].model
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=request_data,
+ input_type="response",
+ logging_obj=litellm_logging_obj,
+ )
+
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+
+ # Step 4: Apply guardrailed text back to all streaming chunks
+ # For each choice, replace the combined text across all chunks
+ await self._apply_guardrail_responses_to_output_streaming(
+ responses=responses_so_far,
+ guardrailed_texts=guardrailed_texts,
+ task_mappings=task_mappings,
+ )
+
+ verbose_proxy_logger.debug(
+ "OpenAI Chat Completions: Processed output streaming responses: %s",
+ responses_so_far,
+ )
+
+ return responses_so_far
+
+ def _combine_streaming_texts(
+ self, responses_so_far: List["ModelResponseStream"]
+ ) -> Dict[Tuple[int, Optional[int]], str]:
+ """
+ Combine all streaming chunks into complete text per choice.
+
+ For streaming, we need to concatenate all delta.content across all chunks.
+
+ Args:
+ responses_so_far: List of LiteLLM ModelResponseStream objects
+
+ Returns:
+ Dict mapping (choice_idx, content_idx) to combined text string
+ """
+ combined_texts: Dict[Tuple[int, Optional[int]], str] = {}
+
+ for response_idx, response in enumerate(responses_so_far):
+ for choice_idx, choice in enumerate(response.choices):
+ if isinstance(choice, litellm.StreamingChoices):
+ content = choice.delta.content
+ elif isinstance(choice, litellm.Choices):
+ content = choice.message.content
+ else:
+ continue
+
+ if content is None:
+ continue
+
+ if isinstance(content, str):
+ # String content - accumulate for this choice
+ str_key: Tuple[int, Optional[int]] = (choice_idx, None)
+ if str_key not in combined_texts:
+ combined_texts[str_key] = ""
+ combined_texts[str_key] += content
+
+ elif isinstance(content, list):
+ # List content - accumulate for each content item
+ for content_idx, content_item in enumerate(content):
+ text_str = content_item.get("text")
+ if text_str:
+ list_key: Tuple[int, Optional[int]] = (
+ choice_idx,
+ content_idx,
+ )
+ if list_key not in combined_texts:
+ combined_texts[list_key] = ""
+ combined_texts[list_key] += text_str
+
+ return combined_texts
+
+ def _has_text_content(
+ self, response: Union["ModelResponse", "ModelResponseStream"]
+ ) -> bool:
+ """
+ Check if response has any text content or tool calls to process.
+
+ Override this method to customize text content detection.
+ """
+ from litellm.types.utils import ModelResponse, ModelResponseStream
+
+ if isinstance(response, ModelResponse):
+ for choice in response.choices:
+ if isinstance(choice, litellm.Choices):
+ # Check for text content
+ if choice.message.content and isinstance(
+ choice.message.content, str
+ ):
+ return True
+ # Check for tool calls
+ if choice.message.tool_calls and isinstance(
+ choice.message.tool_calls, list
+ ):
+ if len(choice.message.tool_calls) > 0:
+ return True
+ elif isinstance(response, ModelResponseStream):
+ for streaming_choice in response.choices:
+ if isinstance(streaming_choice, litellm.StreamingChoices):
+ # Check for text content
+ if streaming_choice.delta.content and isinstance(
+ streaming_choice.delta.content, str
+ ):
+ return True
+ # Check for tool calls
+ if streaming_choice.delta.tool_calls and isinstance(
+ streaming_choice.delta.tool_calls, list
+ ):
+ if len(streaming_choice.delta.tool_calls) > 0:
+ return True
+ return False
+
+ def _extract_output_text_images_and_tool_calls(
+ self,
+ choice: Union[Choices, StreamingChoices],
+ choice_idx: int,
+ texts_to_check: List[str],
+ images_to_check: List[str],
+ tool_calls_to_check: List[Dict[str, Any]],
+ text_task_mappings: List[Tuple[int, Optional[int]]],
+ tool_call_task_mappings: List[Tuple[int, int]],
+ ) -> None:
+ """
+ Extract text content, images, and tool calls from a response choice.
+
+ Override this method to customize text/image/tool call extraction logic.
+ """
+ verbose_proxy_logger.debug(
+ "OpenAI Chat Completions: Processing choice: %s", choice
+ )
+
+ # Determine content source and tool calls based on choice type
+ content = None
+ tool_calls: Optional[List[Any]] = None
+ if isinstance(choice, litellm.Choices):
+ content = choice.message.content
+ tool_calls = choice.message.tool_calls
+ elif isinstance(choice, litellm.StreamingChoices):
+ content = choice.delta.content
+ tool_calls = choice.delta.tool_calls
+ else:
+ # Unknown choice type, skip processing
+ return
+
+ # Process content if it exists
+ if content and isinstance(content, str):
+ # Simple string content
+ texts_to_check.append(content)
+ text_task_mappings.append((choice_idx, None))
+
+ elif content and isinstance(content, list):
+ # List content (e.g., multimodal response)
+ for content_idx, content_item in enumerate(content):
+ # Extract text
+ content_text = content_item.get("text")
+ if content_text:
+ texts_to_check.append(content_text)
+ text_task_mappings.append((choice_idx, int(content_idx)))
+
+ # Extract images
+ if content_item.get("type") == "image_url":
+ image_url = content_item.get("image_url", {})
+ if isinstance(image_url, dict):
+ url = image_url.get("url")
+ if url:
+ images_to_check.append(url)
+
+ # Process tool calls if they exist
+ if tool_calls is not None and isinstance(tool_calls, list):
+ for tool_call_idx, tool_call in enumerate(tool_calls):
+ # Convert tool call to dict format for guardrail processing
+ tool_call_dict = self._convert_tool_call_to_dict(tool_call)
+ if tool_call_dict:
+ tool_calls_to_check.append(tool_call_dict)
+ tool_call_task_mappings.append((choice_idx, int(tool_call_idx)))
+
+ def _convert_tool_call_to_dict(
+ self, tool_call: Union[Dict[str, Any], Any]
+ ) -> Optional[Dict[str, Any]]:
+ """
+ Convert a tool call object to dictionary format.
+
+ Tool calls can be either dict or object depending on the type.
+ """
+ if isinstance(tool_call, dict):
+ return tool_call
+ elif hasattr(tool_call, "id") and hasattr(tool_call, "function"):
+ # Convert object to dict
+ function = tool_call.function
+ function_dict = {}
+ if hasattr(function, "name"):
+ function_dict["name"] = function.name
+ if hasattr(function, "arguments"):
+ function_dict["arguments"] = function.arguments
+
+ tool_call_dict = {
+ "id": tool_call.id if hasattr(tool_call, "id") else None,
+ "type": tool_call.type if hasattr(tool_call, "type") else "function",
+ "function": function_dict,
+ }
+ return tool_call_dict
+ return None
+
+ async def _apply_guardrail_responses_to_output_texts(
+ self,
+ response: "ModelResponse",
+ responses: List[str],
+ task_mappings: List[Tuple[int, Optional[int]]],
+ ) -> None:
+ """
+ Apply guardrail text responses back to output response.
+
+ Override this method to customize how text responses are applied.
+ """
+ for task_idx, guardrail_response in enumerate(responses):
+ mapping = task_mappings[task_idx]
+ choice_idx = cast(int, mapping[0])
+ content_idx_optional = cast(Optional[int], mapping[1])
+
+ choice = cast(Choices, response.choices[choice_idx])
+
+ # Handle content
+ content = choice.message.content
+ if content is None:
+ continue
+
+ if isinstance(content, str) and content_idx_optional is None:
+ # Replace string content with guardrail response
+ choice.message.content = guardrail_response
+
+ elif isinstance(content, list) and content_idx_optional is not None:
+ # Replace specific text item in list content
+ choice.message.content[content_idx_optional]["text"] = guardrail_response # type: ignore
+
+ async def _apply_guardrail_responses_to_output_tool_calls(
+ self,
+ response: "ModelResponse",
+ tool_calls: List[Dict[str, Any]],
+ task_mappings: List[Tuple[int, int]],
+ ) -> None:
+ """
+ Apply guardrailed tool calls back to output response.
+
+ The guardrail may have modified the tool_calls list in place,
+ so we apply the modified tool calls back to the original response.
+
+ Override this method to customize how tool call responses are applied.
+ """
+ for task_idx, (choice_idx, tool_call_idx) in enumerate(task_mappings):
+ if task_idx < len(tool_calls):
+ guardrailed_tool_call = tool_calls[task_idx]
+ choice = cast(Choices, response.choices[choice_idx])
+ choice_tool_calls = choice.message.tool_calls
+
+ if choice_tool_calls is not None and isinstance(
+ choice_tool_calls, list
+ ):
+ if tool_call_idx < len(choice_tool_calls):
+ # Update the tool call with guardrailed version
+ existing_tool_call = choice_tool_calls[tool_call_idx]
+ # Update object attributes (output responses always have typed objects)
+ if "function" in guardrailed_tool_call:
+ func_dict = guardrailed_tool_call["function"]
+ if "arguments" in func_dict:
+ existing_tool_call.function.arguments = func_dict[
+ "arguments"
+ ]
+ if "name" in func_dict:
+ existing_tool_call.function.name = func_dict["name"]
+
+ async def _apply_guardrail_responses_to_output_streaming(
+ self,
+ responses: List["ModelResponseStream"],
+ guardrailed_texts: List[str],
+ task_mappings: List[Tuple[int, Optional[int]]],
+ ) -> None:
+ """
+ Apply guardrail responses back to output streaming responses.
+
+ For streaming responses, the guardrailed text (which is the combined text from all chunks)
+ is placed in the first chunk, and subsequent chunks are cleared.
+
+ Args:
+ responses: List of ModelResponseStream objects to modify
+ guardrailed_texts: List of guardrailed text responses (combined from all chunks)
+ task_mappings: List of tuples (choice_idx, content_idx)
+
+ Override this method to customize how responses are applied to streaming responses.
+ """
+ # Build a mapping of what guardrailed text to use for each (choice_idx, content_idx)
+ guardrail_map: Dict[Tuple[int, Optional[int]], str] = {}
+ for task_idx, guardrail_response in enumerate(guardrailed_texts):
+ mapping = task_mappings[task_idx]
+ choice_idx = cast(int, mapping[0])
+ content_idx_optional = cast(Optional[int], mapping[1])
+ guardrail_map[(choice_idx, content_idx_optional)] = guardrail_response
+
+ # Track which choices we've already set the guardrailed text for
+ # Key: (choice_idx, content_idx), Value: boolean (True if already set)
+ already_set: Dict[Tuple[int, Optional[int]], bool] = {}
+
+ # Iterate through all responses and update content
+ for response_idx, response in enumerate(responses):
+ for choice_idx_in_response, choice in enumerate(response.choices):
+ if isinstance(choice, litellm.StreamingChoices):
+ content = choice.delta.content
+ elif isinstance(choice, litellm.Choices):
+ content = choice.message.content
+ else:
+ continue
+
+ if content is None:
+ continue
+
+ if isinstance(content, str):
+ # String content
+ str_key: Tuple[int, Optional[int]] = (choice_idx_in_response, None)
+ if str_key in guardrail_map:
+ if str_key not in already_set:
+ # First chunk - set the complete guardrailed text
+ if isinstance(choice, litellm.StreamingChoices):
+ choice.delta.content = guardrail_map[str_key]
+ elif isinstance(choice, litellm.Choices):
+ choice.message.content = guardrail_map[str_key]
+ already_set[str_key] = True
+ else:
+ # Subsequent chunks - clear the content
+ if isinstance(choice, litellm.StreamingChoices):
+ choice.delta.content = ""
+ elif isinstance(choice, litellm.Choices):
+ choice.message.content = ""
+
+ elif isinstance(content, list):
+ # List content - handle each content item
+ for content_idx, content_item in enumerate(content):
+ if "text" in content_item:
+ list_key: Tuple[int, Optional[int]] = (
+ choice_idx_in_response,
+ content_idx,
+ )
+ if list_key in guardrail_map:
+ if list_key not in already_set:
+ # First chunk - set the complete guardrailed text
+ content_item["text"] = guardrail_map[list_key]
+ already_set[list_key] = True
+ else:
+ # Subsequent chunks - clear the text
+ content_item["text"] = ""
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/o_series_handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/o_series_handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..d141498cc44716b1a8d97eaa69b29d25528bc0ef
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/o_series_handler.py
@@ -0,0 +1,3 @@
+"""
+LLM Calling done in `openai/openai.py`
+"""
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/o_series_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/o_series_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..fe8aec9bc2bd1b1edbcf87c0bc3ec97080a11757
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/chat/o_series_transformation.py
@@ -0,0 +1,179 @@
+"""
+Support for o1/o3 model family
+
+https://platform.openai.com/docs/guides/reasoning
+
+Translations handled by LiteLLM:
+- modalities: image => drop param (if user opts in to dropping param)
+- role: system ==> translate to role 'user'
+- streaming => faked by LiteLLM
+- Tools, response_format => drop param (if user opts in to dropping param)
+- Logprobs => drop param (if user opts in to dropping param)
+"""
+
+from typing import Any, Coroutine, List, Literal, Optional, Union, cast, overload
+
+import litellm
+from litellm import verbose_logger
+from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
+from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage
+from litellm.utils import (
+ supports_function_calling,
+ supports_parallel_function_calling,
+ supports_response_schema,
+ supports_system_messages,
+)
+
+from .gpt_transformation import OpenAIGPTConfig
+
+
+class OpenAIOSeriesConfig(OpenAIGPTConfig):
+ """
+ Reference: https://platform.openai.com/docs/guides/reasoning
+ """
+
+ @classmethod
+ def get_config(cls):
+ return super().get_config()
+
+ def translate_developer_role_to_system_role(
+ self, messages: List[AllMessageValues]
+ ) -> List[AllMessageValues]:
+ """
+ O-series models support `developer` role.
+ """
+ return messages
+
+ def get_supported_openai_params(self, model: str) -> list:
+ """
+ Get the supported OpenAI params for the given model
+
+ """
+
+ all_openai_params = super().get_supported_openai_params(model=model)
+ non_supported_params = [
+ "logprobs",
+ "top_p",
+ "presence_penalty",
+ "frequency_penalty",
+ "top_logprobs",
+ ]
+
+ o_series_only_param = ["reasoning_effort"]
+
+ all_openai_params.extend(o_series_only_param)
+
+ try:
+ model, custom_llm_provider, api_base, api_key = get_llm_provider(
+ model=model
+ )
+ except Exception:
+ verbose_logger.debug(
+ f"Unable to infer model provider for model={model}, defaulting to openai for o1 supported param check"
+ )
+ custom_llm_provider = "openai"
+
+ _supports_function_calling = supports_function_calling(
+ model, custom_llm_provider
+ )
+ _supports_response_schema = supports_response_schema(model, custom_llm_provider)
+ _supports_parallel_tool_calls = supports_parallel_function_calling(
+ model, custom_llm_provider
+ )
+
+ if not _supports_function_calling:
+ non_supported_params.append("tools")
+ non_supported_params.append("tool_choice")
+ non_supported_params.append("function_call")
+ non_supported_params.append("functions")
+
+ if not _supports_parallel_tool_calls:
+ non_supported_params.append("parallel_tool_calls")
+
+ if not _supports_response_schema:
+ non_supported_params.append("response_format")
+
+ return [
+ param for param in all_openai_params if param not in non_supported_params
+ ]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ):
+ if "max_tokens" in non_default_params:
+ optional_params["max_completion_tokens"] = non_default_params.pop(
+ "max_tokens"
+ )
+ if "temperature" in non_default_params:
+ temperature_value: Optional[float] = non_default_params.pop("temperature")
+ if temperature_value is not None:
+ if temperature_value == 1:
+ optional_params["temperature"] = temperature_value
+ else:
+ ## UNSUPPORTED TOOL CHOICE VALUE
+ if litellm.drop_params is True or drop_params is True:
+ pass
+ else:
+ raise litellm.utils.UnsupportedParamsError(
+ message="O-series models don't support temperature={}. Only temperature=1 is supported. To drop unsupported openai params from the call, set `litellm.drop_params = True`".format(
+ temperature_value
+ ),
+ status_code=400,
+ )
+
+ return super()._map_openai_params(
+ non_default_params, optional_params, model, drop_params
+ )
+
+ def is_model_o_series_model(self, model: str) -> bool:
+ model = model.split("/")[-1] # could be "openai/o3" or "o3"
+ return (
+ len(model) > 1
+ and model[0] == "o"
+ and model[1].isdigit()
+ and model in litellm.open_ai_chat_completion_models
+ )
+
+ @overload
+ def _transform_messages(
+ self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
+ ) -> Coroutine[Any, Any, List[AllMessageValues]]:
+ ...
+
+ @overload
+ def _transform_messages(
+ self,
+ messages: List[AllMessageValues],
+ model: str,
+ is_async: Literal[False] = False,
+ ) -> List[AllMessageValues]:
+ ...
+
+ def _transform_messages(
+ self, messages: List[AllMessageValues], model: str, is_async: bool = False
+ ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
+ """
+ Handles limitations of O-1 model family.
+ - modalities: image => drop param (if user opts in to dropping param)
+ - role: system ==> translate to role 'user'
+ """
+ _supports_system_messages = supports_system_messages(model, "openai")
+ for i, message in enumerate(messages):
+ if message["role"] == "system" and not _supports_system_messages:
+ new_message = ChatCompletionUserMessage(
+ content=message["content"], role="user"
+ )
+ messages[i] = new_message # Replace the old message with the new one
+
+ if is_async:
+ return super()._transform_messages(
+ messages, model, is_async=cast(Literal[True], True)
+ )
+ else:
+ return super()._transform_messages(
+ messages, model, is_async=cast(Literal[False], False)
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/common_utils.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/common_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..35723ccd637f75f05b90c55d0cee8fe7d761e527
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/common_utils.py
@@ -0,0 +1,290 @@
+"""
+Common helpers / utils across al OpenAI endpoints
+"""
+
+import hashlib
+import inspect
+import json
+import os
+import ssl
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ Dict,
+ List,
+ Literal,
+ NamedTuple,
+ Optional,
+ Tuple,
+ Union,
+)
+
+import httpx
+import openai
+from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
+
+if TYPE_CHECKING:
+ from aiohttp import ClientSession
+
+import litellm
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.custom_httpx.http_handler import (
+ _DEFAULT_TTL_FOR_HTTPX_CLIENTS,
+ AsyncHTTPHandler,
+ get_ssl_configuration,
+)
+
+
+def _get_client_init_params(cls: type) -> Tuple[str, ...]:
+ """Extract __init__ parameter names (excluding 'self') from a class."""
+ return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self") # type: ignore[misc]
+
+
+_OPENAI_INIT_PARAMS: Tuple[str, ...] = _get_client_init_params(OpenAI)
+_AZURE_OPENAI_INIT_PARAMS: Tuple[str, ...] = _get_client_init_params(AzureOpenAI)
+
+
+class OpenAIError(BaseLLMException):
+ def __init__(
+ self,
+ status_code: int,
+ message: str,
+ request: Optional[httpx.Request] = None,
+ response: Optional[httpx.Response] = None,
+ headers: Optional[Union[dict, httpx.Headers]] = None,
+ body: Optional[dict] = None,
+ ):
+ self.status_code = status_code
+ self.message = message
+ self.headers = headers
+ if request:
+ self.request = request
+ else:
+ self.request = httpx.Request(method="POST", url="https://api.openai.com/v1")
+ if response:
+ self.response = response
+ else:
+ self.response = httpx.Response(
+ status_code=status_code, request=self.request
+ )
+ super().__init__(
+ status_code=status_code,
+ message=self.message,
+ headers=self.headers,
+ request=self.request,
+ response=self.response,
+ body=body,
+ )
+
+
+####### Error Handling Utils for OpenAI API #######################
+###################################################################
+def drop_params_from_unprocessable_entity_error(
+ e: Union[openai.UnprocessableEntityError, httpx.HTTPStatusError],
+ data: Dict[str, Any],
+) -> Dict[str, Any]:
+ """
+ Helper function to read OpenAI UnprocessableEntityError and drop the params that raised an error from the error message.
+
+ Args:
+ e (UnprocessableEntityError): The UnprocessableEntityError exception
+ data (Dict[str, Any]): The original data dictionary containing all parameters
+
+ Returns:
+ Dict[str, Any]: A new dictionary with invalid parameters removed
+ """
+ invalid_params: List[str] = []
+ if isinstance(e, httpx.HTTPStatusError):
+ error_json = e.response.json()
+ error_message = error_json.get("error", {})
+ error_body = error_message
+ else:
+ error_body = e.body
+ if (
+ error_body is not None
+ and isinstance(error_body, dict)
+ and error_body.get("message")
+ ):
+ message = error_body.get("message", {})
+ if isinstance(message, str):
+ try:
+ message = json.loads(message)
+ except json.JSONDecodeError:
+ message = {"detail": message}
+ detail = message.get("detail")
+
+ if isinstance(detail, List) and len(detail) > 0 and isinstance(detail[0], dict):
+ for error_dict in detail:
+ if (
+ error_dict.get("loc")
+ and isinstance(error_dict.get("loc"), list)
+ and len(error_dict.get("loc")) == 2
+ ):
+ invalid_params.append(error_dict["loc"][1])
+
+ new_data = {k: v for k, v in data.items() if k not in invalid_params}
+
+ return new_data
+
+
+class BaseOpenAILLM:
+ """
+ Base class for OpenAI LLMs for getting their httpx clients and SSL verification settings
+ """
+
+ @staticmethod
+ def get_cached_openai_client(
+ client_initialization_params: dict, client_type: Literal["openai", "azure"]
+ ) -> Optional[Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]]:
+ """Retrieves the OpenAI client from the in-memory cache based on the client initialization parameters"""
+ _cache_key = BaseOpenAILLM.get_openai_client_cache_key(
+ client_initialization_params=client_initialization_params,
+ client_type=client_type,
+ )
+ _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_cache_key)
+ return _cached_client
+
+ @staticmethod
+ def set_cached_openai_client(
+ openai_client: Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI],
+ client_type: Literal["openai", "azure"],
+ client_initialization_params: dict,
+ ):
+ """Stores the OpenAI client in the in-memory cache for _DEFAULT_TTL_FOR_HTTPX_CLIENTS SECONDS"""
+ _cache_key = BaseOpenAILLM.get_openai_client_cache_key(
+ client_initialization_params=client_initialization_params,
+ client_type=client_type,
+ )
+ litellm.in_memory_llm_clients_cache.set_cache(
+ key=_cache_key,
+ value=openai_client,
+ ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
+ )
+
+ @staticmethod
+ def get_openai_client_cache_key(
+ client_initialization_params: dict, client_type: Literal["openai", "azure"]
+ ) -> str:
+ """Creates a cache key for the OpenAI client based on the client initialization parameters"""
+ hashed_api_key = None
+ if client_initialization_params.get("api_key") is not None:
+ hash_object = hashlib.sha256(
+ client_initialization_params.get("api_key", "").encode()
+ )
+ # Hexadecimal representation of the hash
+ hashed_api_key = hash_object.hexdigest()
+
+ # Create a more readable cache key using a list of key-value pairs
+ key_parts = [
+ f"hashed_api_key={hashed_api_key}",
+ f"is_async={client_initialization_params.get('is_async')}",
+ ]
+
+ LITELLM_CLIENT_SPECIFIC_PARAMS = (
+ "timeout",
+ "max_retries",
+ "organization",
+ "api_base",
+ )
+ openai_client_fields = (
+ BaseOpenAILLM.get_openai_client_initialization_param_fields(
+ client_type=client_type
+ )
+ + LITELLM_CLIENT_SPECIFIC_PARAMS
+ )
+
+ for param in openai_client_fields:
+ key_parts.append(f"{param}={client_initialization_params.get(param)}")
+
+ _cache_key = ",".join(key_parts)
+ return _cache_key
+
+ @staticmethod
+ def get_openai_client_initialization_param_fields(
+ client_type: Literal["openai", "azure"]
+ ) -> Tuple[str, ...]:
+ """Returns a tuple of fields that are used to initialize the OpenAI client"""
+ if client_type == "openai":
+ return _OPENAI_INIT_PARAMS
+ else:
+ return _AZURE_OPENAI_INIT_PARAMS
+
+ @staticmethod
+ def _get_async_http_client(
+ shared_session: Optional["ClientSession"] = None,
+ ) -> Optional[httpx.AsyncClient]:
+ if litellm.aclient_session is not None:
+ return litellm.aclient_session
+
+ if getattr(litellm, "network_mock", False):
+ from litellm.llms.custom_httpx.mock_transport import MockOpenAITransport
+
+ return httpx.AsyncClient(transport=MockOpenAITransport())
+
+ # Get unified SSL configuration
+ ssl_config = get_ssl_configuration()
+
+ return httpx.AsyncClient(
+ verify=ssl_config,
+ transport=AsyncHTTPHandler._create_async_transport(
+ ssl_context=ssl_config
+ if isinstance(ssl_config, ssl.SSLContext)
+ else None,
+ ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
+ shared_session=shared_session,
+ ),
+ follow_redirects=True,
+ )
+
+ @staticmethod
+ def _get_sync_http_client() -> Optional[httpx.Client]:
+ if litellm.client_session is not None:
+ return litellm.client_session
+
+ if getattr(litellm, "network_mock", False):
+ from litellm.llms.custom_httpx.mock_transport import MockOpenAITransport
+
+ return httpx.Client(transport=MockOpenAITransport())
+
+ # Get unified SSL configuration
+ ssl_config = get_ssl_configuration()
+
+ return httpx.Client(
+ verify=ssl_config,
+ follow_redirects=True,
+ )
+
+
+class OpenAICredentials(NamedTuple):
+ api_base: str
+ api_key: Optional[str]
+ organization: Optional[str]
+
+
+def get_openai_credentials(
+ api_base: Optional[str] = None,
+ api_key: Optional[str] = None,
+ organization: Optional[str] = None,
+) -> OpenAICredentials:
+ """Resolve OpenAI credentials from params, litellm globals, and env vars."""
+ resolved_api_base = (
+ api_base
+ or litellm.api_base
+ or os.getenv("OPENAI_BASE_URL")
+ or os.getenv("OPENAI_API_BASE")
+ or "https://api.openai.com/v1"
+ )
+ resolved_organization = (
+ organization
+ or litellm.organization
+ or os.getenv("OPENAI_ORGANIZATION", None)
+ or None
+ )
+ resolved_api_key = (
+ api_key or litellm.api_key or litellm.openai_key or os.getenv("OPENAI_API_KEY")
+ )
+ return OpenAICredentials(
+ api_base=resolved_api_base,
+ api_key=resolved_api_key,
+ organization=resolved_organization,
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/guardrail_translation/README.md b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/guardrail_translation/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..93762206c47d771883b9b997790b9d6156fb7100
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/guardrail_translation/README.md
@@ -0,0 +1,158 @@
+# OpenAI Text Completion Guardrail Translation Handler
+
+Handler for processing OpenAI's text completion endpoint (`/v1/completions`) with guardrails.
+
+## Overview
+
+This handler processes text completion requests by:
+1. Extracting the text prompt(s) from the request
+2. Applying guardrails to the prompt text(s)
+3. Updating the request with the guardrailed prompt(s)
+4. Applying guardrails to the completion output text
+
+## Data Format
+
+### Input Format
+
+**Single Prompt:**
+```json
+{
+ "model": "gpt-3.5-turbo-instruct",
+ "prompt": "Say this is a test",
+ "max_tokens": 7,
+ "temperature": 0
+}
+```
+
+**Multiple Prompts (Batch):**
+```json
+{
+ "model": "gpt-3.5-turbo-instruct",
+ "prompt": [
+ "Tell me a joke",
+ "Write a poem"
+ ],
+ "max_tokens": 50
+}
+```
+
+### Output Format
+
+```json
+{
+ "id": "cmpl-uqkvlQyYK7bGYrRHQ0eXlWi7",
+ "object": "text_completion",
+ "created": 1589478378,
+ "model": "gpt-3.5-turbo-instruct",
+ "choices": [
+ {
+ "text": "\n\nThis is indeed a test",
+ "index": 0,
+ "logprobs": null,
+ "finish_reason": "length"
+ }
+ ],
+ "usage": {
+ "prompt_tokens": 5,
+ "completion_tokens": 7,
+ "total_tokens": 12
+ }
+}
+```
+
+## Usage
+
+The handler is automatically discovered and applied when guardrails are used with the text completion endpoint.
+
+### Example: Using Guardrails with Text Completion
+
+```bash
+curl -X POST 'http://localhost:4000/v1/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer your-api-key' \
+-d '{
+ "model": "gpt-3.5-turbo-instruct",
+ "prompt": "Say this is a test",
+ "guardrails": ["content_moderation"],
+ "max_tokens": 7
+}'
+```
+
+The guardrail will be applied to both:
+- **Input**: The prompt text before sending to the LLM
+- **Output**: The completion text in the response
+
+### Example: PII Masking in Prompts and Completions
+
+```bash
+curl -X POST 'http://localhost:4000/v1/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer your-api-key' \
+-d '{
+ "model": "gpt-3.5-turbo-instruct",
+ "prompt": "My name is John Doe and my email is john@example.com",
+ "guardrails": ["mask_pii"],
+ "metadata": {
+ "guardrails": ["mask_pii"]
+ }
+}'
+```
+
+### Example: Batch Prompts with Guardrails
+
+```bash
+curl -X POST 'http://localhost:4000/v1/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer your-api-key' \
+-d '{
+ "model": "gpt-3.5-turbo-instruct",
+ "prompt": [
+ "Tell me about AI",
+ "What is machine learning?"
+ ],
+ "guardrails": ["content_filter"],
+ "max_tokens": 100
+}'
+```
+
+## Implementation Details
+
+### Input Processing
+
+- **Field**: `prompt` (string or list of strings)
+- **Processing**:
+ - String prompts: Apply guardrail directly
+ - List prompts: Apply guardrail to each string in the list
+- **Result**: Updated prompt(s) in request
+
+### Output Processing
+
+- **Field**: `choices[*].text` (string)
+- **Processing**: Applies guardrail to each completion text
+- **Result**: Updated completion texts in response
+
+### Supported Prompt Types
+
+1. **String**: Single prompt as a string
+2. **List of Strings**: Multiple prompts for batch completion
+3. **List of Lists**: Token-based prompts (passed through unchanged)
+
+## Extension
+
+Override these methods to customize behavior:
+
+- `process_input_messages()`: Customize how prompts are processed
+- `process_output_response()`: Customize how completion texts are processed
+
+## Supported Call Types
+
+- `CallTypes.text_completion` - Synchronous text completion
+- `CallTypes.atext_completion` - Asynchronous text completion
+
+## Notes
+
+- The handler processes both input prompts and output completion texts
+- List prompts are processed individually (each string in the list)
+- Non-string prompt items (e.g., token lists) are passed through unchanged
+- Both sync and async call types use the same handler
+
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/guardrail_translation/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/guardrail_translation/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..51e43c459379f50de223207a56ffd76fa2899681
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/guardrail_translation/__init__.py
@@ -0,0 +1,13 @@
+"""OpenAI Text Completion handler for Unified Guardrails."""
+
+from litellm.llms.openai.completion.guardrail_translation.handler import (
+ OpenAITextCompletionHandler,
+)
+from litellm.types.utils import CallTypes
+
+guardrail_translation_mappings = {
+ CallTypes.text_completion: OpenAITextCompletionHandler,
+ CallTypes.atext_completion: OpenAITextCompletionHandler,
+}
+
+__all__ = ["guardrail_translation_mappings", "OpenAITextCompletionHandler"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/guardrail_translation/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/guardrail_translation/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..1f8c6159da03b0a9868f91a13624eb2b1a68c640
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/guardrail_translation/handler.py
@@ -0,0 +1,194 @@
+"""
+OpenAI Text Completion Handler for Unified Guardrails
+
+This module provides guardrail translation support for OpenAI's text completion endpoint.
+The handler processes the 'prompt' parameter for guardrails.
+"""
+
+from typing import TYPE_CHECKING, Any, Optional
+
+from litellm._logging import verbose_proxy_logger
+from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
+from litellm.types.utils import GenericGuardrailAPIInputs
+
+if TYPE_CHECKING:
+ from litellm.integrations.custom_guardrail import CustomGuardrail
+ from litellm.types.utils import TextCompletionResponse
+
+
+class OpenAITextCompletionHandler(BaseTranslation):
+ """
+ Handler for processing OpenAI text completion requests with guardrails.
+
+ This class provides methods to:
+ 1. Process input prompt (pre-call hook)
+ 2. Process output response (post-call hook)
+
+ The handler specifically processes the 'prompt' parameter which can be:
+ - A single string
+ - A list of strings (for batch completions)
+ """
+
+ async def process_input_messages(
+ self,
+ data: dict,
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process input prompt by applying guardrails to text content.
+
+ Args:
+ data: Request data dictionary containing 'prompt' parameter
+ guardrail_to_apply: The guardrail instance to apply
+
+ Returns:
+ Modified data with guardrails applied to prompt
+ """
+ prompt = data.get("prompt")
+ if prompt is None:
+ verbose_proxy_logger.debug(
+ "OpenAI Text Completion: No prompt found in request data"
+ )
+ return data
+
+ if isinstance(prompt, str):
+ # Single string prompt
+ inputs = GenericGuardrailAPIInputs(texts=[prompt])
+ # Include model information if available
+ model = data.get("model")
+ if model:
+ inputs["model"] = model
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=data,
+ input_type="request",
+ logging_obj=litellm_logging_obj,
+ )
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+ data["prompt"] = guardrailed_texts[0] if guardrailed_texts else prompt
+
+ verbose_proxy_logger.debug(
+ "OpenAI Text Completion: Applied guardrail to string prompt. "
+ "Original length: %d, New length: %d",
+ len(prompt),
+ len(data["prompt"]),
+ )
+
+ elif isinstance(prompt, list):
+ # List of string prompts (batch completion)
+ texts_to_check = []
+ text_indices = [] # Track which prompts are strings
+
+ for idx, p in enumerate(prompt):
+ if isinstance(p, str):
+ texts_to_check.append(p)
+ text_indices.append(idx)
+
+ if texts_to_check:
+ inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
+ # Include model information if available
+ model = data.get("model")
+ if model:
+ inputs["model"] = model
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=data,
+ input_type="request",
+ logging_obj=litellm_logging_obj,
+ )
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+
+ # Replace guardrailed texts back
+ for guardrail_idx, prompt_idx in enumerate(text_indices):
+ if guardrail_idx < len(guardrailed_texts):
+ data["prompt"][prompt_idx] = guardrailed_texts[guardrail_idx]
+ verbose_proxy_logger.debug(
+ "OpenAI Text Completion: Applied guardrail to prompt[%d]. "
+ "Original length: %d, New length: %d",
+ prompt_idx,
+ len(texts_to_check[guardrail_idx]),
+ len(guardrailed_texts[guardrail_idx]),
+ )
+
+ else:
+ verbose_proxy_logger.warning(
+ "OpenAI Text Completion: Unexpected prompt type: %s. Expected string or list.",
+ type(prompt),
+ )
+
+ return data
+
+ async def process_output_response(
+ self,
+ response: "TextCompletionResponse",
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ user_api_key_dict: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process output response by applying guardrails to completion text.
+
+ Args:
+ response: Text completion response object
+ guardrail_to_apply: The guardrail instance to apply
+ litellm_logging_obj: Optional logging object
+ user_api_key_dict: User API key metadata to pass to guardrails
+
+ Returns:
+ Modified response with guardrails applied to completion text
+ """
+ if not hasattr(response, "choices") or not response.choices:
+ verbose_proxy_logger.debug(
+ "OpenAI Text Completion: No choices in response to process"
+ )
+ return response
+
+ # Collect all texts to check
+ texts_to_check = []
+ choice_indices = []
+
+ for idx, choice in enumerate(response.choices):
+ if hasattr(choice, "text") and isinstance(choice.text, str):
+ texts_to_check.append(choice.text)
+ choice_indices.append(idx)
+
+ # Apply guardrails in batch
+ if texts_to_check:
+ # Create a request_data dict with response info and user API key metadata
+ request_data: dict = {"response": response}
+
+ # Add user API key metadata with prefixed keys
+ user_metadata = self.transform_user_api_key_dict_to_metadata(
+ user_api_key_dict
+ )
+ if user_metadata:
+ request_data["litellm_metadata"] = user_metadata
+
+ inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
+ # Include model information from the response if available
+ if hasattr(response, "model") and response.model:
+ inputs["model"] = response.model
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=request_data,
+ input_type="response",
+ logging_obj=litellm_logging_obj,
+ )
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+
+ # Apply guardrailed texts back to choices
+ for guardrail_idx, choice_idx in enumerate(choice_indices):
+ if guardrail_idx < len(guardrailed_texts):
+ original_text = response.choices[choice_idx].text
+ response.choices[choice_idx].text = guardrailed_texts[guardrail_idx]
+
+ verbose_proxy_logger.debug(
+ "OpenAI Text Completion: Applied guardrail to choice[%d] text. "
+ "Original length: %d, New length: %d",
+ choice_idx,
+ len(original_text),
+ len(guardrailed_texts[guardrail_idx]),
+ )
+
+ return response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..1641615126e842455d403beaba598691864002c6
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/handler.py
@@ -0,0 +1,318 @@
+import json
+from typing import Callable, List, Optional, Union
+
+from openai import AsyncOpenAI, OpenAI
+
+import litellm
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
+from litellm.llms.base import BaseLLM
+from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage
+from litellm.types.utils import LlmProviders, ModelResponse, TextCompletionResponse
+from litellm.utils import ProviderConfigManager
+
+from ..common_utils import BaseOpenAILLM, OpenAIError
+from .transformation import OpenAITextCompletionConfig
+
+
+class OpenAITextCompletion(BaseLLM):
+ openai_text_completion_global_config = OpenAITextCompletionConfig()
+
+ def __init__(self) -> None:
+ super().__init__()
+
+ def validate_environment(self, api_key):
+ headers = {
+ "content-type": "application/json",
+ }
+ if api_key:
+ headers["Authorization"] = f"Bearer {api_key}"
+ return headers
+
+ def completion(
+ self,
+ model_response: ModelResponse,
+ api_key: str,
+ model: str,
+ messages: Union[List[AllMessageValues], List[OpenAITextCompletionUserMessage]],
+ timeout: float,
+ custom_llm_provider: str,
+ logging_obj: LiteLLMLoggingObj,
+ optional_params: dict,
+ print_verbose: Optional[Callable] = None,
+ api_base: Optional[str] = None,
+ acompletion: bool = False,
+ litellm_params=None,
+ logger_fn=None,
+ client=None,
+ organization: Optional[str] = None,
+ headers: Optional[dict] = None,
+ ):
+ try:
+ if headers is None:
+ headers = self.validate_environment(api_key=api_key)
+ if model is None or messages is None:
+ raise OpenAIError(status_code=422, message="Missing model or messages")
+
+ # don't send max retries to the api, if set
+
+ provider_config = ProviderConfigManager.get_provider_text_completion_config(
+ model=model,
+ provider=LlmProviders(custom_llm_provider),
+ )
+
+ data = provider_config.transform_text_completion_request(
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ headers=headers,
+ )
+ max_retries = data.pop("max_retries", 2)
+ ## LOGGING
+ logging_obj.pre_call(
+ input=messages,
+ api_key=api_key,
+ additional_args={
+ "headers": headers,
+ "api_base": api_base,
+ "complete_input_dict": data,
+ },
+ )
+ if acompletion is True:
+ if optional_params.get("stream", False):
+ return self.async_streaming(
+ logging_obj=logging_obj,
+ api_base=api_base,
+ api_key=api_key,
+ data=data,
+ headers=headers,
+ model_response=model_response,
+ model=model,
+ timeout=timeout,
+ max_retries=max_retries,
+ client=client,
+ organization=organization,
+ )
+ else:
+ return self.acompletion(api_base=api_base, data=data, headers=headers, model_response=model_response, api_key=api_key, logging_obj=logging_obj, model=model, timeout=timeout, max_retries=max_retries, organization=organization, client=client) # type: ignore
+ elif optional_params.get("stream", False):
+ return self.streaming(
+ logging_obj=logging_obj,
+ api_base=api_base,
+ api_key=api_key,
+ data=data,
+ headers=headers,
+ model_response=model_response,
+ model=model,
+ timeout=timeout,
+ max_retries=max_retries, # type: ignore
+ client=client,
+ organization=organization,
+ )
+ else:
+ if client is None:
+ openai_client = OpenAI(
+ api_key=api_key,
+ base_url=api_base,
+ http_client=litellm.client_session,
+ timeout=timeout,
+ max_retries=max_retries, # type: ignore
+ organization=organization,
+ )
+ else:
+ openai_client = client
+
+ raw_response = openai_client.completions.with_raw_response.create(**data) # type: ignore
+ response = raw_response.parse()
+ response_json = response.model_dump()
+
+ ## LOGGING
+ logging_obj.post_call(
+ api_key=api_key,
+ original_response=response_json,
+ additional_args={
+ "headers": headers,
+ "api_base": api_base,
+ },
+ )
+
+ ## RESPONSE OBJECT
+ return TextCompletionResponse(**response_json)
+ except Exception as e:
+ status_code = getattr(e, "status_code", 500)
+ error_headers = getattr(e, "headers", None)
+ error_text = getattr(e, "text", str(e))
+ error_response = getattr(e, "response", None)
+ if error_headers is None and error_response:
+ error_headers = getattr(error_response, "headers", None)
+ raise OpenAIError(
+ status_code=status_code, message=error_text, headers=error_headers
+ )
+
+ async def acompletion(
+ self,
+ logging_obj,
+ api_base: str,
+ data: dict,
+ headers: dict,
+ model_response: ModelResponse,
+ api_key: str,
+ model: str,
+ timeout: float,
+ max_retries: int,
+ organization: Optional[str] = None,
+ client=None,
+ ):
+ try:
+ if client is None:
+ openai_aclient = AsyncOpenAI(
+ api_key=api_key,
+ base_url=api_base,
+ http_client=BaseOpenAILLM._get_async_http_client(),
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ )
+ else:
+ openai_aclient = client
+
+ raw_response = await openai_aclient.completions.with_raw_response.create(
+ **data
+ )
+ response = raw_response.parse()
+ response_json = response.model_dump()
+
+ ## LOGGING
+ logging_obj.post_call(
+ api_key=api_key,
+ original_response=response,
+ additional_args={
+ "headers": headers,
+ "api_base": api_base,
+ },
+ )
+ ## RESPONSE OBJECT
+ response_obj = TextCompletionResponse(**response_json)
+ response_obj._hidden_params.original_response = json.dumps(response_json)
+ return response_obj
+ except Exception as e:
+ status_code = getattr(e, "status_code", 500)
+ error_headers = getattr(e, "headers", None)
+ error_text = getattr(e, "text", str(e))
+ error_response = getattr(e, "response", None)
+ if error_headers is None and error_response:
+ error_headers = getattr(error_response, "headers", None)
+ raise OpenAIError(
+ status_code=status_code, message=error_text, headers=error_headers
+ )
+
+ def streaming(
+ self,
+ logging_obj,
+ api_key: str,
+ data: dict,
+ headers: dict,
+ model_response: ModelResponse,
+ model: str,
+ timeout: float,
+ api_base: Optional[str] = None,
+ max_retries=None,
+ client=None,
+ organization=None,
+ ):
+ if client is None:
+ openai_client = OpenAI(
+ api_key=api_key,
+ base_url=api_base,
+ http_client=litellm.client_session,
+ timeout=timeout,
+ max_retries=max_retries, # type: ignore
+ organization=organization,
+ )
+ else:
+ openai_client = client
+
+ try:
+ raw_response = openai_client.completions.with_raw_response.create(**data)
+ response = raw_response.parse()
+ except Exception as e:
+ status_code = getattr(e, "status_code", 500)
+ error_headers = getattr(e, "headers", None)
+ error_text = getattr(e, "text", str(e))
+ error_response = getattr(e, "response", None)
+ if error_headers is None and error_response:
+ error_headers = getattr(error_response, "headers", None)
+ raise OpenAIError(
+ status_code=status_code, message=error_text, headers=error_headers
+ )
+ streamwrapper = CustomStreamWrapper(
+ completion_stream=response,
+ model=model,
+ custom_llm_provider="text-completion-openai",
+ logging_obj=logging_obj,
+ stream_options=data.get("stream_options", None),
+ )
+
+ try:
+ for chunk in streamwrapper:
+ yield chunk
+ except Exception as e:
+ status_code = getattr(e, "status_code", 500)
+ error_headers = getattr(e, "headers", None)
+ error_text = getattr(e, "text", str(e))
+ error_response = getattr(e, "response", None)
+ if error_headers is None and error_response:
+ error_headers = getattr(error_response, "headers", None)
+ raise OpenAIError(
+ status_code=status_code, message=error_text, headers=error_headers
+ )
+
+ async def async_streaming(
+ self,
+ logging_obj,
+ api_key: str,
+ data: dict,
+ headers: dict,
+ model_response: ModelResponse,
+ model: str,
+ timeout: float,
+ max_retries: int,
+ api_base: Optional[str] = None,
+ client=None,
+ organization=None,
+ ):
+ if client is None:
+ openai_client = AsyncOpenAI(
+ api_key=api_key,
+ base_url=api_base,
+ http_client=litellm.aclient_session,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ )
+ else:
+ openai_client = client
+
+ raw_response = await openai_client.completions.with_raw_response.create(**data)
+ response = raw_response.parse()
+ streamwrapper = CustomStreamWrapper(
+ completion_stream=response,
+ model=model,
+ custom_llm_provider="text-completion-openai",
+ logging_obj=logging_obj,
+ stream_options=data.get("stream_options", None),
+ )
+
+ try:
+ async for transformed_chunk in streamwrapper:
+ yield transformed_chunk
+ except Exception as e:
+ status_code = getattr(e, "status_code", 500)
+ error_headers = getattr(e, "headers", None)
+ error_text = getattr(e, "text", str(e))
+ error_response = getattr(e, "response", None)
+ if error_headers is None and error_response:
+ error_headers = getattr(error_response, "headers", None)
+ raise OpenAIError(
+ status_code=status_code, message=error_text, headers=error_headers
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..44a4949d4552deb6bc5dc8fe7ec1fb3fa7b9148d
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/transformation.py
@@ -0,0 +1,158 @@
+"""
+Support for gpt model family
+"""
+
+from typing import List, Optional, Union
+
+from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig
+from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage
+from litellm.types.utils import Choices, Message, ModelResponse, TextCompletionResponse
+
+from ..chat.gpt_transformation import OpenAIGPTConfig
+from .utils import _transform_prompt
+
+
+class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig):
+ """
+ Reference: https://platform.openai.com/docs/api-reference/completions/create
+
+ The class `OpenAITextCompletionConfig` provides configuration for the OpenAI's text completion API interface. Below are the parameters:
+
+ - `best_of` (integer or null): This optional parameter generates server-side completions and returns the one with the highest log probability per token.
+
+ - `echo` (boolean or null): This optional parameter will echo back the prompt in addition to the completion.
+
+ - `frequency_penalty` (number or null): Defaults to 0. It is a numbers from -2.0 to 2.0, where positive values decrease the model's likelihood to repeat the same line.
+
+ - `logit_bias` (map): This optional parameter modifies the likelihood of specified tokens appearing in the completion.
+
+ - `logprobs` (integer or null): This optional parameter includes the log probabilities on the most likely tokens as well as the chosen tokens.
+
+ - `max_tokens` (integer or null): This optional parameter sets the maximum number of tokens to generate in the completion.
+
+ - `n` (integer or null): This optional parameter sets how many completions to generate for each prompt.
+
+ - `presence_penalty` (number or null): Defaults to 0 and can be between -2.0 and 2.0. Positive values increase the model's likelihood to talk about new topics.
+
+ - `stop` (string / array / null): Specifies up to 4 sequences where the API will stop generating further tokens.
+
+ - `suffix` (string or null): Defines the suffix that comes after a completion of inserted text.
+
+ - `temperature` (number or null): This optional parameter defines the sampling temperature to use.
+
+ - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling.
+ """
+
+ best_of: Optional[int] = None
+ echo: Optional[bool] = None
+ frequency_penalty: Optional[int] = None
+ logit_bias: Optional[dict] = None
+ logprobs: Optional[int] = None
+ max_tokens: Optional[int] = None
+ n: Optional[int] = None
+ presence_penalty: Optional[int] = None
+ stop: Optional[Union[str, list]] = None
+ suffix: Optional[str] = None
+
+ def __init__(
+ self,
+ best_of: Optional[int] = None,
+ echo: Optional[bool] = None,
+ frequency_penalty: Optional[int] = None,
+ logit_bias: Optional[dict] = None,
+ logprobs: Optional[int] = None,
+ max_tokens: Optional[int] = None,
+ n: Optional[int] = None,
+ presence_penalty: Optional[int] = None,
+ stop: Optional[Union[str, list]] = None,
+ suffix: Optional[str] = None,
+ temperature: Optional[float] = None,
+ top_p: Optional[float] = None,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return super().get_config()
+
+ def convert_to_chat_model_response_object(
+ self,
+ response_object: Optional[TextCompletionResponse] = None,
+ model_response_object: Optional[ModelResponse] = None,
+ ):
+ try:
+ ## RESPONSE OBJECT
+ if response_object is None or model_response_object is None:
+ raise ValueError("Error in response object format")
+ choice_list: List[Choices] = []
+ for idx, choice in enumerate(response_object["choices"]):
+ message = Message(
+ content=choice["text"],
+ role="assistant",
+ )
+ choice = Choices(
+ finish_reason=choice["finish_reason"],
+ index=idx,
+ message=message,
+ logprobs=choice.get("logprobs", None),
+ )
+ choice_list.append(choice)
+ model_response_object.choices = choice_list # type: ignore
+
+ if "usage" in response_object:
+ setattr(model_response_object, "usage", response_object["usage"])
+
+ if "id" in response_object:
+ model_response_object.id = response_object["id"]
+
+ if "model" in response_object:
+ model_response_object.model = response_object["model"]
+
+ model_response_object._hidden_params[
+ "original_response"
+ ] = response_object # track original response, if users make a litellm.text_completion() request, we can return the original response
+ return model_response_object
+ except Exception as e:
+ raise e
+
+ def get_supported_openai_params(self, model: str) -> List:
+ return [
+ "functions",
+ "function_call",
+ "temperature",
+ "top_p",
+ "n",
+ "stream",
+ "stream_options",
+ "stop",
+ "max_tokens",
+ "presence_penalty",
+ "frequency_penalty",
+ "logit_bias",
+ "user",
+ "response_format",
+ "seed",
+ "tools",
+ "tool_choice",
+ "max_retries",
+ "logprobs",
+ "top_logprobs",
+ "extra_headers",
+ ]
+
+ def transform_text_completion_request(
+ self,
+ model: str,
+ messages: Union[List[AllMessageValues], List[OpenAITextCompletionUserMessage]],
+ optional_params: dict,
+ headers: dict,
+ ) -> dict:
+ prompt = _transform_prompt(messages)
+ return {
+ "model": model,
+ "prompt": prompt,
+ **optional_params,
+ }
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/utils.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..8b3efb4cda8524d92f370b8e9712f4b32e770a6e
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/completion/utils.py
@@ -0,0 +1,50 @@
+from typing import List, Union, cast
+
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+ convert_content_list_to_str,
+)
+from litellm.types.llms.openai import (
+ AllMessageValues,
+ AllPromptValues,
+ OpenAITextCompletionUserMessage,
+)
+
+
+def is_tokens_or_list_of_tokens(value: List):
+ # Check if it's a list of integers (tokens)
+ if isinstance(value, list) and all(isinstance(item, int) for item in value):
+ return True
+ # Check if it's a list of lists of integers (list of tokens)
+ if isinstance(value, list) and all(
+ isinstance(item, list) and all(isinstance(i, int) for i in item)
+ for item in value
+ ):
+ return True
+ return False
+
+
+def _transform_prompt(
+ messages: Union[List[AllMessageValues], List[OpenAITextCompletionUserMessage]],
+) -> AllPromptValues:
+ if len(messages) == 1: # base case
+ message_content = messages[0].get("content")
+ if (
+ message_content
+ and isinstance(message_content, list)
+ and is_tokens_or_list_of_tokens(message_content)
+ ):
+ openai_prompt: AllPromptValues = cast(AllPromptValues, message_content)
+ else:
+ openai_prompt = ""
+ content = convert_content_list_to_str(cast(AllMessageValues, messages[0]))
+ openai_prompt += content
+ else:
+ prompt_str_list: List[str] = []
+ for m in messages:
+ try: # expect list of int/list of list of int to be a 1 message array only.
+ content = convert_content_list_to_str(cast(AllMessageValues, m))
+ prompt_str_list.append(content)
+ except Exception as e:
+ raise e
+ openai_prompt = prompt_str_list
+ return openai_prompt
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/containers/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/containers/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..645538fdd9c617adc6d3a7bfb4580a21fd9d9ad3
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/containers/transformation.py
@@ -0,0 +1,343 @@
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
+
+import httpx
+
+import litellm
+from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
+ StandardBuiltInToolCostTracking,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.containers.main import (
+ ContainerCreateOptionalRequestParams,
+ ContainerFileListResponse,
+ ContainerListResponse,
+ ContainerObject,
+ DeleteContainerResult,
+)
+from litellm.types.router import GenericLiteLLMParams
+
+from ...base_llm.containers.transformation import BaseContainerConfig
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+ from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException
+
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
+ BaseLLMException = _BaseLLMException
+else:
+ LiteLLMLoggingObj = Any
+ BaseLLMException = Any
+
+
+class OpenAIContainerConfig(BaseContainerConfig):
+ """Configuration class for OpenAI container API."""
+
+ def __init__(self):
+ super().__init__()
+
+ def get_supported_openai_params(self) -> list:
+ """Get the list of supported OpenAI parameters for container API."""
+ return [
+ "name",
+ "expires_after",
+ "file_ids",
+ "extra_headers",
+ ]
+
+ def map_openai_params(
+ self,
+ container_create_optional_params: ContainerCreateOptionalRequestParams,
+ drop_params: bool,
+ ) -> Dict:
+ """No mapping applied since inputs are in OpenAI spec already"""
+ return dict(container_create_optional_params)
+
+ def validate_environment(
+ self,
+ headers: dict,
+ api_key: Optional[str] = None,
+ ) -> dict:
+ api_key = (
+ api_key
+ or litellm.api_key
+ or litellm.openai_key
+ or get_secret_str("OPENAI_API_KEY")
+ )
+ headers.update(
+ {
+ "Authorization": f"Bearer {api_key}",
+ },
+ )
+ return headers
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ litellm_params: dict,
+ ) -> str:
+ """Get the complete URL for OpenAI container API."""
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("OPENAI_BASE_URL")
+ or get_secret_str("OPENAI_API_BASE")
+ or "https://api.openai.com/v1"
+ )
+
+ return f"{api_base.rstrip('/')}/containers"
+
+ def transform_container_create_request(
+ self,
+ name: str,
+ container_create_optional_request_params: Dict,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Dict:
+ """Transform the container creation request for OpenAI API."""
+ # Remove extra_headers from optional params as they're handled separately
+ container_create_optional_request_params = {
+ k: v
+ for k, v in container_create_optional_request_params.items()
+ if k not in ["extra_headers"]
+ }
+
+ # Create the request data
+ request_dict = {
+ "name": name,
+ **container_create_optional_request_params,
+ }
+
+ return request_dict
+
+ def transform_container_create_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ContainerObject:
+ """Transform the OpenAI container creation response."""
+ response_data = raw_response.json()
+
+ # Transform the response data
+ container_obj = ContainerObject(**response_data) # type: ignore[arg-type]
+
+ # Add cost for container creation (OpenAI containers are code interpreter sessions)
+ # https://platform.openai.com/docs/pricing
+ # Each container creation is 1 code interpreter session
+ container_cost = StandardBuiltInToolCostTracking.get_cost_for_code_interpreter(
+ sessions=1,
+ provider="openai",
+ )
+
+ if (
+ not hasattr(container_obj, "_hidden_params")
+ or container_obj._hidden_params is None
+ ):
+ container_obj._hidden_params = {}
+ if "additional_headers" not in container_obj._hidden_params:
+ container_obj._hidden_params["additional_headers"] = {}
+ container_obj._hidden_params["additional_headers"][
+ "llm_provider-x-litellm-response-cost"
+ ] = container_cost
+
+ return container_obj
+
+ def transform_container_list_request(
+ self,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ after: Optional[str] = None,
+ limit: Optional[int] = None,
+ order: Optional[str] = None,
+ extra_query: Optional[Dict[str, Any]] = None,
+ ) -> Tuple[str, Dict]:
+ """Transform the container list request for OpenAI API.
+
+ OpenAI API expects the following request:
+ - GET /v1/containers
+ """
+ # Use the api_base directly for container list
+ url = api_base
+
+ # Prepare query parameters
+ params = {}
+ if after is not None:
+ params["after"] = after
+ if limit is not None:
+ params["limit"] = str(limit)
+ if order is not None:
+ params["order"] = order
+
+ # Add any extra query parameters
+ if extra_query:
+ params.update(extra_query)
+
+ return url, params
+
+ def transform_container_list_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ContainerListResponse:
+ """Transform the OpenAI container list response."""
+ response_data = raw_response.json()
+
+ # Transform the response data
+ container_list = ContainerListResponse(**response_data) # type: ignore[arg-type]
+
+ return container_list
+
+ def transform_container_retrieve_request(
+ self,
+ container_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """Transform the OpenAI container retrieve request."""
+ # For container retrieve, we just need to construct the URL
+ url = f"{api_base.rstrip('/')}/{container_id}"
+
+ # No additional data needed for GET request
+ data: Dict[str, Any] = {}
+
+ return url, data
+
+ def transform_container_retrieve_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ContainerObject:
+ """Transform the OpenAI container retrieve response."""
+ response_data = raw_response.json()
+ # Transform the response data
+ container_obj = ContainerObject(**response_data) # type: ignore[arg-type]
+
+ return container_obj
+
+ def transform_container_delete_request(
+ self,
+ container_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """Transform the container delete request for OpenAI API.
+
+ OpenAI API expects the following request:
+ - DELETE /v1/containers/{container_id}
+ """
+ # Construct the URL for container delete
+ url = f"{api_base.rstrip('/')}/{container_id}"
+
+ # No data needed for DELETE request
+ data: Dict[str, Any] = {}
+
+ return url, data
+
+ def transform_container_delete_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> DeleteContainerResult:
+ """Transform the OpenAI container delete response."""
+ response_data = raw_response.json()
+
+ # Transform the response data
+ delete_result = DeleteContainerResult(**response_data) # type: ignore[arg-type]
+
+ return delete_result
+
+ def transform_container_file_list_request(
+ self,
+ container_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ after: Optional[str] = None,
+ limit: Optional[int] = None,
+ order: Optional[str] = None,
+ extra_query: Optional[Dict[str, Any]] = None,
+ ) -> Tuple[str, Dict]:
+ """Transform the container file list request for OpenAI API.
+
+ OpenAI API expects the following request:
+ - GET /v1/containers/{container_id}/files
+ """
+ # Construct the URL for container files
+ url = f"{api_base.rstrip('/')}/{container_id}/files"
+
+ # Prepare query parameters
+ params: Dict[str, Any] = {}
+ if after is not None:
+ params["after"] = after
+ if limit is not None:
+ params["limit"] = str(limit)
+ if order is not None:
+ params["order"] = order
+
+ # Add any extra query parameters
+ if extra_query:
+ params.update(extra_query)
+
+ return url, params
+
+ def transform_container_file_list_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ContainerFileListResponse:
+ """Transform the OpenAI container file list response."""
+ response_data = raw_response.json()
+
+ # Transform the response data
+ file_list = ContainerFileListResponse(**response_data) # type: ignore[arg-type]
+
+ return file_list
+
+ def transform_container_file_content_request(
+ self,
+ container_id: str,
+ file_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """Transform the container file content request for OpenAI API.
+
+ OpenAI API expects the following request:
+ - GET /v1/containers/{container_id}/files/{file_id}/content
+ """
+ # Construct the URL for container file content
+ url = f"{api_base.rstrip('/')}/{container_id}/files/{file_id}/content"
+
+ # No query parameters needed
+ params: Dict[str, Any] = {}
+
+ return url, params
+
+ def transform_container_file_content_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> bytes:
+ """Transform the OpenAI container file content response.
+
+ Returns the raw binary content of the file.
+ """
+ return raw_response.content
+
+ def get_error_class(
+ self,
+ error_message: str,
+ status_code: int,
+ headers: Union[dict, httpx.Headers],
+ ) -> BaseLLMException:
+ from ...base_llm.chat.transformation import BaseLLMException
+
+ raise BaseLLMException(
+ status_code=status_code,
+ message=error_message,
+ headers=headers,
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/cost_calculation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/cost_calculation.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac1e4a6b08f8af46aec407868bcaff8b028fa9af
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/cost_calculation.py
@@ -0,0 +1,177 @@
+"""
+Helper util for handling openai-specific cost calculation
+- e.g.: prompt caching
+"""
+
+from typing import Literal, Optional, Tuple
+
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
+from litellm.types.utils import CallTypes, ModelInfo, Usage
+from litellm.utils import get_model_info
+
+
+def cost_router(call_type: CallTypes) -> Literal["cost_per_token", "cost_per_second"]:
+ if call_type == CallTypes.atranscription or call_type == CallTypes.transcription:
+ return "cost_per_second"
+ else:
+ return "cost_per_token"
+
+
+def cost_per_token(
+ model: str, usage: Usage, service_tier: Optional[str] = None
+) -> Tuple[float, float]:
+ """
+ Calculates the cost per token for a given model, prompt tokens, and completion tokens.
+
+ Input:
+ - model: str, the model name without provider prefix
+ - usage: LiteLLM Usage block, containing anthropic caching information
+
+ Returns:
+ Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
+ """
+ ## CALCULATE INPUT COST
+ return generic_cost_per_token(
+ model=model,
+ usage=usage,
+ custom_llm_provider="openai",
+ service_tier=service_tier,
+ )
+ # ### Non-cached text tokens
+ # non_cached_text_tokens = usage.prompt_tokens
+ # cached_tokens: Optional[int] = None
+ # if usage.prompt_tokens_details and usage.prompt_tokens_details.cached_tokens:
+ # cached_tokens = usage.prompt_tokens_details.cached_tokens
+ # non_cached_text_tokens = non_cached_text_tokens - cached_tokens
+ # prompt_cost: float = non_cached_text_tokens * model_info["input_cost_per_token"]
+ # ## Prompt Caching cost calculation
+ # if model_info.get("cache_read_input_token_cost") is not None and cached_tokens:
+ # # Note: We read ._cache_read_input_tokens from the Usage - since cost_calculator.py standardizes the cache read tokens on usage._cache_read_input_tokens
+ # prompt_cost += cached_tokens * (
+ # model_info.get("cache_read_input_token_cost", 0) or 0
+ # )
+
+ # _audio_tokens: Optional[int] = (
+ # usage.prompt_tokens_details.audio_tokens
+ # if usage.prompt_tokens_details is not None
+ # else None
+ # )
+ # _audio_cost_per_token: Optional[float] = model_info.get(
+ # "input_cost_per_audio_token"
+ # )
+ # if _audio_tokens is not None and _audio_cost_per_token is not None:
+ # audio_cost: float = _audio_tokens * _audio_cost_per_token
+ # prompt_cost += audio_cost
+
+ # ## CALCULATE OUTPUT COST
+ # completion_cost: float = (
+ # usage["completion_tokens"] * model_info["output_cost_per_token"]
+ # )
+ # _output_cost_per_audio_token: Optional[float] = model_info.get(
+ # "output_cost_per_audio_token"
+ # )
+ # _output_audio_tokens: Optional[int] = (
+ # usage.completion_tokens_details.audio_tokens
+ # if usage.completion_tokens_details is not None
+ # else None
+ # )
+ # if _output_cost_per_audio_token is not None and _output_audio_tokens is not None:
+ # audio_cost = _output_audio_tokens * _output_cost_per_audio_token
+ # completion_cost += audio_cost
+
+ # return prompt_cost, completion_cost
+
+
+def cost_per_second(
+ model: str, custom_llm_provider: Optional[str], duration: float = 0.0
+) -> Tuple[float, float]:
+ """
+ Calculates the cost per second for a given model, prompt tokens, and completion tokens.
+
+ Input:
+ - model: str, the model name without provider prefix
+ - custom_llm_provider: str, the custom llm provider
+ - duration: float, the duration of the response in seconds
+
+ Returns:
+ Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
+ """
+
+ ## GET MODEL INFO
+ model_info = get_model_info(
+ model=model, custom_llm_provider=custom_llm_provider or "openai"
+ )
+ prompt_cost = 0.0
+ completion_cost = 0.0
+ ## Speech / Audio cost calculation
+ if (
+ "output_cost_per_second" in model_info
+ and model_info["output_cost_per_second"] is not None
+ ):
+ verbose_logger.debug(
+ f"For model={model} - output_cost_per_second: {model_info.get('output_cost_per_second')}; duration: {duration}"
+ )
+ ## COST PER SECOND ##
+ completion_cost = model_info["output_cost_per_second"] * duration
+ elif (
+ "input_cost_per_second" in model_info
+ and model_info["input_cost_per_second"] is not None
+ ):
+ verbose_logger.debug(
+ f"For model={model} - input_cost_per_second: {model_info.get('input_cost_per_second')}; duration: {duration}"
+ )
+ ## COST PER SECOND ##
+ prompt_cost = model_info["input_cost_per_second"] * duration
+ completion_cost = 0.0
+
+ return prompt_cost, completion_cost
+
+
+def video_generation_cost(
+ model: str,
+ duration_seconds: float,
+ custom_llm_provider: Optional[str] = None,
+ model_info: Optional[ModelInfo] = None,
+) -> float:
+ """
+ Calculates the cost for video generation based on duration in seconds.
+
+ Input:
+ - model: str, the model name without provider prefix
+ - duration_seconds: float, the duration of the generated video in seconds
+ - custom_llm_provider: str, the custom llm provider
+ - model_info: Optional[dict], deployment-level model info containing
+ custom video pricing. When provided, skips the global
+ get_model_info() lookup so that deployment-specific pricing is used.
+
+ Returns:
+ float - total_cost_in_usd
+ """
+ ## GET MODEL INFO
+ if model_info is None:
+ model_info = get_model_info(
+ model=model, custom_llm_provider=custom_llm_provider or "openai"
+ )
+
+ # Check for video-specific cost per second
+ video_cost_per_second = model_info.get("output_cost_per_video_per_second")
+ if video_cost_per_second is not None:
+ verbose_logger.debug(
+ f"For model={model} - output_cost_per_video_per_second: {video_cost_per_second}; duration: {duration_seconds}"
+ )
+ return video_cost_per_second * duration_seconds
+
+ # Fallback to general output cost per second
+ output_cost_per_second = model_info.get("output_cost_per_second")
+ if output_cost_per_second is not None:
+ verbose_logger.debug(
+ f"For model={model} - output_cost_per_second: {output_cost_per_second}; duration: {duration_seconds}"
+ )
+ return output_cost_per_second * duration_seconds
+
+ # If no cost information found, return 0
+ verbose_logger.warning(
+ f"No cost information found for video model {model}. Please add pricing to model_prices_and_context_window.json"
+ )
+ return 0.0
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/embeddings/guardrail_translation/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/embeddings/guardrail_translation/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..a60662282ca96711c28ce9dbbfe3c7fdcbabecbb
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/embeddings/guardrail_translation/__init__.py
@@ -0,0 +1,13 @@
+"""OpenAI Embeddings handler for Unified Guardrails."""
+
+from litellm.llms.openai.embeddings.guardrail_translation.handler import (
+ OpenAIEmbeddingsHandler,
+)
+from litellm.types.utils import CallTypes
+
+guardrail_translation_mappings = {
+ CallTypes.embedding: OpenAIEmbeddingsHandler,
+ CallTypes.aembedding: OpenAIEmbeddingsHandler,
+}
+
+__all__ = ["guardrail_translation_mappings", "OpenAIEmbeddingsHandler"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/embeddings/guardrail_translation/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/embeddings/guardrail_translation/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..7458020e109b624ddd9f26984fd3f92615942b53
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/embeddings/guardrail_translation/handler.py
@@ -0,0 +1,179 @@
+"""
+OpenAI Embeddings Handler for Unified Guardrails
+
+This module provides guardrail translation support for OpenAI's embeddings endpoint.
+The handler processes the 'input' parameter for guardrails.
+"""
+
+from typing import TYPE_CHECKING, Any, List, Optional, Union
+
+from litellm._logging import verbose_proxy_logger
+from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
+from litellm.types.utils import GenericGuardrailAPIInputs
+
+if TYPE_CHECKING:
+ from litellm.integrations.custom_guardrail import CustomGuardrail
+ from litellm.types.utils import EmbeddingResponse
+
+
+class OpenAIEmbeddingsHandler(BaseTranslation):
+ """
+ Handler for processing OpenAI embeddings requests with guardrails.
+
+ This class provides methods to:
+ 1. Process input text (pre-call hook)
+ 2. Process output response (post-call hook) - embeddings don't typically need output guardrails
+
+ The handler specifically processes the 'input' parameter which can be:
+ - A single string
+ - A list of strings (for batch embeddings)
+ - A list of integers (token IDs - not processed by guardrails)
+ - A list of lists of integers (batch token IDs - not processed by guardrails)
+ """
+
+ async def process_input_messages(
+ self,
+ data: dict,
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process input text by applying guardrails to text content.
+
+ Args:
+ data: Request data dictionary containing 'input' parameter
+ guardrail_to_apply: The guardrail instance to apply
+ litellm_logging_obj: Optional logging object
+
+ Returns:
+ Modified data with guardrails applied to input
+ """
+ input_data = data.get("input")
+ if input_data is None:
+ verbose_proxy_logger.debug(
+ "OpenAI Embeddings: No input found in request data"
+ )
+ return data
+
+ if isinstance(input_data, str):
+ data = await self._process_string_input(
+ data, input_data, guardrail_to_apply, litellm_logging_obj
+ )
+ elif isinstance(input_data, list):
+ data = await self._process_list_input(
+ data, input_data, guardrail_to_apply, litellm_logging_obj
+ )
+ else:
+ verbose_proxy_logger.warning(
+ "OpenAI Embeddings: Unexpected input type: %s. Expected string or list.",
+ type(input_data),
+ )
+
+ return data
+
+ async def _process_string_input(
+ self,
+ data: dict,
+ input_data: str,
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any],
+ ) -> dict:
+ """Process a single string input through the guardrail."""
+ inputs = GenericGuardrailAPIInputs(texts=[input_data])
+ if model := data.get("model"):
+ inputs["model"] = model
+
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=data,
+ input_type="request",
+ logging_obj=litellm_logging_obj,
+ )
+
+ if guardrailed_texts := guardrailed_inputs.get("texts"):
+ data["input"] = guardrailed_texts[0]
+ verbose_proxy_logger.debug(
+ "OpenAI Embeddings: Applied guardrail to string input. "
+ "Original length: %d, New length: %d",
+ len(input_data),
+ len(data["input"]),
+ )
+
+ return data
+
+ async def _process_list_input(
+ self,
+ data: dict,
+ input_data: List[Union[str, int, List[int]]],
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any],
+ ) -> dict:
+ """Process a list input through the guardrail (if it contains strings)."""
+ if len(input_data) == 0:
+ return data
+
+ first_item = input_data[0]
+
+ # Skip non-text inputs (token IDs)
+ if isinstance(first_item, (int, list)):
+ verbose_proxy_logger.debug(
+ "OpenAI Embeddings: Input is token IDs, skipping guardrail processing"
+ )
+ return data
+
+ if not isinstance(first_item, str):
+ verbose_proxy_logger.warning(
+ "OpenAI Embeddings: Unexpected input list item type: %s",
+ type(first_item),
+ )
+ return data
+
+ # List of strings - apply guardrail
+ inputs = GenericGuardrailAPIInputs(texts=input_data) # type: ignore
+ if model := data.get("model"):
+ inputs["model"] = model
+
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=data,
+ input_type="request",
+ logging_obj=litellm_logging_obj,
+ )
+
+ if guardrailed_texts := guardrailed_inputs.get("texts"):
+ data["input"] = guardrailed_texts
+ verbose_proxy_logger.debug(
+ "OpenAI Embeddings: Applied guardrail to %d inputs",
+ len(guardrailed_texts),
+ )
+
+ return data
+
+ async def process_output_response(
+ self,
+ response: "EmbeddingResponse",
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ user_api_key_dict: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process output response - embeddings responses contain vectors, not text.
+
+ For embeddings, the output is numerical vectors, so there's typically
+ no text content to apply guardrails to. This method is a no-op but
+ is included for interface consistency.
+
+ Args:
+ response: Embedding response object
+ guardrail_to_apply: The guardrail instance to apply
+ litellm_logging_obj: Optional logging object
+ user_api_key_dict: User API key metadata
+
+ Returns:
+ Unmodified response (embeddings don't have text output to guard)
+ """
+ verbose_proxy_logger.debug(
+ "OpenAI Embeddings: Output response processing skipped - "
+ "embeddings contain vectors, not text"
+ )
+ return response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/evals/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/evals/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b04d27622bbd8ce5d65f9361e48e4b19f0e21679
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/evals/__init__.py
@@ -0,0 +1,7 @@
+"""
+OpenAI Evals API configuration
+"""
+
+from .transformation import OpenAIEvalsConfig
+
+__all__ = ["OpenAIEvalsConfig"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/evals/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/evals/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..c24dbf8637a80113be0096748be61be2eed80e0a
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/evals/transformation.py
@@ -0,0 +1,426 @@
+"""
+OpenAI Evals API configuration and transformations
+"""
+
+from typing import Any, Dict, Optional, Tuple
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.llms.base_llm.evals.transformation import (
+ BaseEvalsAPIConfig,
+ LiteLLMLoggingObj,
+)
+from litellm.types.llms.openai_evals import (
+ CancelEvalResponse,
+ CancelRunResponse,
+ CreateEvalRequest,
+ CreateRunRequest,
+ DeleteEvalResponse,
+ Eval,
+ ListEvalsParams,
+ ListEvalsResponse,
+ ListRunsParams,
+ ListRunsResponse,
+ Run,
+ RunDeleteResponse,
+ UpdateEvalRequest,
+)
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import LlmProviders
+
+
+class OpenAIEvalsConfig(BaseEvalsAPIConfig):
+ """OpenAI-specific Evals API configuration"""
+
+ @property
+ def custom_llm_provider(self) -> LlmProviders:
+ return LlmProviders.OPENAI
+
+ def validate_environment(
+ self, headers: dict, litellm_params: Optional[GenericLiteLLMParams]
+ ) -> dict:
+ """Add OpenAI-specific headers"""
+ import litellm
+ from litellm.secret_managers.main import get_secret_str
+
+ # Get API key following OpenAI pattern
+ api_key = None
+ if litellm_params:
+ api_key = litellm_params.api_key
+
+ api_key = (
+ api_key
+ or litellm.api_key
+ or litellm.openai_key
+ or get_secret_str("OPENAI_API_KEY")
+ )
+
+ if not api_key:
+ raise ValueError("OPENAI_API_KEY is required for Evals API")
+
+ # Add required headers
+ headers["Authorization"] = f"Bearer {api_key}"
+ headers["Content-Type"] = "application/json"
+
+ return headers
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ endpoint: str,
+ eval_id: Optional[str] = None,
+ ) -> str:
+ """Get complete URL for OpenAI Evals API"""
+ if api_base is None:
+ api_base = "https://api.openai.com"
+
+ if eval_id:
+ return f"{api_base}/v1/evals/{eval_id}"
+ return f"{api_base}/v1/{endpoint}"
+
+ def transform_create_eval_request(
+ self,
+ create_request: CreateEvalRequest,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Dict:
+ """Transform create eval request for OpenAI"""
+ verbose_logger.debug("Transforming create eval request: %s", create_request)
+
+ # OpenAI expects the request body directly
+ request_body = {k: v for k, v in create_request.items() if v is not None}
+
+ return request_body
+
+ def transform_create_eval_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> Eval:
+ """Transform OpenAI response to Eval object"""
+ response_json = raw_response.json()
+ verbose_logger.debug("Transforming create eval response: %s", response_json)
+
+ return Eval(**response_json)
+
+ def transform_list_evals_request(
+ self,
+ list_params: ListEvalsParams,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """Transform list evals request for OpenAI"""
+ api_base = "https://api.openai.com"
+ if litellm_params and litellm_params.api_base:
+ api_base = litellm_params.api_base
+
+ url = self.get_complete_url(api_base=api_base, endpoint="evals")
+
+ # Build query parameters
+ query_params: Dict[str, Any] = {}
+ if "limit" in list_params and list_params["limit"]:
+ query_params["limit"] = list_params["limit"]
+ if "after" in list_params and list_params["after"]:
+ query_params["after"] = list_params["after"]
+ if "before" in list_params and list_params["before"]:
+ query_params["before"] = list_params["before"]
+ if "order" in list_params and list_params["order"]:
+ query_params["order"] = list_params["order"]
+ if "order_by" in list_params and list_params["order_by"]:
+ query_params["order_by"] = list_params["order_by"]
+
+ verbose_logger.debug(
+ "List evals request made to OpenAI Evals endpoint with params: %s",
+ query_params,
+ )
+
+ return url, query_params
+
+ def transform_list_evals_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ListEvalsResponse:
+ """Transform OpenAI response to ListEvalsResponse"""
+ response_json = raw_response.json()
+ verbose_logger.debug("Transforming list evals response: %s", response_json)
+
+ return ListEvalsResponse(**response_json)
+
+ def transform_get_eval_request(
+ self,
+ eval_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """Transform get eval request for OpenAI"""
+ url = self.get_complete_url(
+ api_base=api_base, endpoint="evals", eval_id=eval_id
+ )
+
+ verbose_logger.debug("Get eval request - URL: %s", url)
+
+ return url, headers
+
+ def transform_get_eval_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> Eval:
+ """Transform OpenAI response to Eval object"""
+ response_json = raw_response.json()
+ verbose_logger.debug("Transforming get eval response: %s", response_json)
+
+ return Eval(**response_json)
+
+ def transform_update_eval_request(
+ self,
+ eval_id: str,
+ update_request: UpdateEvalRequest,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict, Dict]:
+ """Transform update eval request for OpenAI"""
+ url = self.get_complete_url(
+ api_base=api_base, endpoint="evals", eval_id=eval_id
+ )
+
+ # Build request body
+ request_body = {k: v for k, v in update_request.items() if v is not None}
+
+ verbose_logger.debug(
+ "Update eval request - URL: %s, body: %s", url, request_body
+ )
+
+ return url, headers, request_body
+
+ def transform_update_eval_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> Eval:
+ """Transform OpenAI response to Eval object"""
+ response_json = raw_response.json()
+ verbose_logger.debug("Transforming update eval response: %s", response_json)
+
+ return Eval(**response_json)
+
+ def transform_delete_eval_request(
+ self,
+ eval_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """Transform delete eval request for OpenAI"""
+ url = self.get_complete_url(
+ api_base=api_base, endpoint="evals", eval_id=eval_id
+ )
+
+ verbose_logger.debug("Delete eval request - URL: %s", url)
+
+ return url, headers
+
+ def transform_delete_eval_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> DeleteEvalResponse:
+ """Transform OpenAI response to DeleteEvalResponse"""
+ response_json = raw_response.json()
+ verbose_logger.debug("Transforming delete eval response: %s", response_json)
+
+ return DeleteEvalResponse(**response_json)
+
+ def transform_cancel_eval_request(
+ self,
+ eval_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict, Dict]:
+ """Transform cancel eval request for OpenAI"""
+ url = f"{self.get_complete_url(api_base=api_base, endpoint='evals', eval_id=eval_id)}/cancel"
+
+ # Empty body for cancel request
+ request_body: Dict[str, Any] = {}
+
+ verbose_logger.debug("Cancel eval request - URL: %s", url)
+
+ return url, headers, request_body
+
+ def transform_cancel_eval_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> CancelEvalResponse:
+ """Transform OpenAI response to CancelEvalResponse"""
+ response_json = raw_response.json()
+ verbose_logger.debug("Transforming cancel eval response: %s", response_json)
+
+ return CancelEvalResponse(**response_json)
+
+ # Run API Transformations
+ def transform_create_run_request(
+ self,
+ eval_id: str,
+ create_request: CreateRunRequest,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """Transform create run request for OpenAI"""
+ api_base = "https://api.openai.com"
+ if litellm_params and litellm_params.api_base:
+ api_base = litellm_params.api_base
+
+ url = f"{api_base}/v1/evals/{eval_id}/runs"
+
+ # Build request body
+ request_body = {k: v for k, v in create_request.items() if v is not None}
+
+ verbose_logger.debug(
+ "Create run request - URL: %s, body: %s", url, request_body
+ )
+
+ return url, request_body
+
+ def transform_create_run_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> Run:
+ """Transform OpenAI response to Run object"""
+ response_json = raw_response.json()
+ verbose_logger.debug("Transforming create run response: %s", response_json)
+
+ return Run(**response_json)
+
+ def transform_list_runs_request(
+ self,
+ eval_id: str,
+ list_params: ListRunsParams,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """Transform list runs request for OpenAI"""
+ api_base = "https://api.openai.com"
+ if litellm_params and litellm_params.api_base:
+ api_base = litellm_params.api_base
+
+ url = f"{api_base}/v1/evals/{eval_id}/runs"
+
+ # Build query parameters
+ query_params: Dict[str, Any] = {}
+ if "limit" in list_params and list_params["limit"]:
+ query_params["limit"] = list_params["limit"]
+ if "after" in list_params and list_params["after"]:
+ query_params["after"] = list_params["after"]
+ if "before" in list_params and list_params["before"]:
+ query_params["before"] = list_params["before"]
+ if "order" in list_params and list_params["order"]:
+ query_params["order"] = list_params["order"]
+
+ verbose_logger.debug(
+ "List runs request made to OpenAI Evals endpoint with params: %s",
+ query_params,
+ )
+
+ return url, query_params
+
+ def transform_list_runs_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ListRunsResponse:
+ """Transform OpenAI response to ListRunsResponse"""
+ response_json = raw_response.json()
+ verbose_logger.debug("Transforming list runs response: %s", response_json)
+
+ return ListRunsResponse(**response_json)
+
+ def transform_get_run_request(
+ self,
+ eval_id: str,
+ run_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """Transform get run request for OpenAI"""
+ url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
+
+ verbose_logger.debug("Get run request - URL: %s", url)
+
+ return url, headers
+
+ def transform_get_run_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> Run:
+ """Transform OpenAI response to Run object"""
+ response_json = raw_response.json()
+ verbose_logger.debug("Transforming get run response: %s", response_json)
+
+ return Run(**response_json)
+
+ def transform_cancel_run_request(
+ self,
+ eval_id: str,
+ run_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict, Dict]:
+ """Transform cancel run request for OpenAI"""
+ url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}/cancel"
+
+ # Empty body for cancel request
+ request_body: Dict[str, Any] = {}
+
+ verbose_logger.debug("Cancel run request - URL: %s", url)
+
+ return url, headers, request_body
+
+ def transform_cancel_run_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> CancelRunResponse:
+ """Transform OpenAI response to CancelRunResponse"""
+ response_json = raw_response.json()
+ verbose_logger.debug("Transforming cancel run response: %s", response_json)
+
+ return CancelRunResponse(**response_json)
+
+ def transform_delete_run_request(
+ self,
+ eval_id: str,
+ run_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict, Dict]:
+ """Transform delete run request for OpenAI"""
+ url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
+
+ # Empty body for delete request
+ request_body: Dict[str, Any] = {}
+
+ verbose_logger.debug("Delete run request - URL: %s", url)
+
+ return url, headers, request_body
+
+ def transform_delete_run_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> RunDeleteResponse:
+ """Transform OpenAI response to RunDeleteResponse"""
+ response_json = raw_response.json()
+ verbose_logger.debug("Transforming delete run response: %s", response_json)
+
+ return RunDeleteResponse(**response_json)
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/fine_tuning/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/fine_tuning/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..9804ff3539e6e4effc6a17e6a0451d1897500300
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/fine_tuning/handler.py
@@ -0,0 +1,278 @@
+from typing import Any, Coroutine, Optional, Union, cast
+
+import httpx
+from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
+
+from litellm._logging import verbose_logger
+from litellm.types.utils import LiteLLMFineTuningJob
+
+
+class OpenAIFineTuningAPI:
+ """
+ OpenAI methods to support for batches
+ """
+
+ def __init__(self) -> None:
+ super().__init__()
+
+ def get_openai_client(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[
+ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
+ ] = None,
+ _is_async: bool = False,
+ api_version: Optional[str] = None,
+ litellm_params: Optional[dict] = None,
+ ) -> Optional[Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI,]]:
+ received_args = locals()
+ openai_client: Optional[
+ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
+ ] = None
+ if client is None:
+ data = {}
+ for k, v in received_args.items():
+ if k == "self" or k == "client" or k == "_is_async":
+ pass
+ elif k == "api_base" and v is not None:
+ data["base_url"] = v
+ elif v is not None:
+ data[k] = v
+ if _is_async is True:
+ openai_client = AsyncOpenAI(**data)
+ else:
+ openai_client = OpenAI(**data) # type: ignore
+ else:
+ openai_client = client
+
+ return openai_client
+
+ async def acreate_fine_tuning_job(
+ self,
+ create_fine_tuning_job_data: dict,
+ openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI],
+ ) -> LiteLLMFineTuningJob:
+ response = await openai_client.fine_tuning.jobs.create(
+ **create_fine_tuning_job_data
+ )
+
+ return LiteLLMFineTuningJob(**response.model_dump())
+
+ def create_fine_tuning_job(
+ self,
+ _is_async: bool,
+ create_fine_tuning_job_data: dict,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ api_version: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[
+ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
+ ] = None,
+ ) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
+ openai_client: Optional[
+ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
+ ] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ api_version=api_version,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, (AsyncOpenAI, AsyncAzureOpenAI)):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.acreate_fine_tuning_job( # type: ignore
+ create_fine_tuning_job_data=create_fine_tuning_job_data,
+ openai_client=openai_client,
+ )
+ verbose_logger.debug(
+ "creating fine tuning job, args= %s", create_fine_tuning_job_data
+ )
+ response = cast(OpenAI, openai_client).fine_tuning.jobs.create(
+ **create_fine_tuning_job_data
+ )
+ return LiteLLMFineTuningJob(**response.model_dump())
+
+ async def acancel_fine_tuning_job(
+ self,
+ fine_tuning_job_id: str,
+ openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI],
+ ) -> LiteLLMFineTuningJob:
+ response = await openai_client.fine_tuning.jobs.cancel(
+ fine_tuning_job_id=fine_tuning_job_id
+ )
+ return LiteLLMFineTuningJob(**response.model_dump())
+
+ def cancel_fine_tuning_job(
+ self,
+ _is_async: bool,
+ fine_tuning_job_id: str,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ api_version: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[
+ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
+ ] = None,
+ ) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
+ openai_client: Optional[
+ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
+ ] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ api_version=api_version,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, (AsyncOpenAI, AsyncAzureOpenAI)):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.acancel_fine_tuning_job( # type: ignore
+ fine_tuning_job_id=fine_tuning_job_id,
+ openai_client=openai_client,
+ )
+ verbose_logger.debug("canceling fine tuning job, args= %s", fine_tuning_job_id)
+ response = cast(OpenAI, openai_client).fine_tuning.jobs.cancel(
+ fine_tuning_job_id=fine_tuning_job_id
+ )
+ return LiteLLMFineTuningJob(**response.model_dump())
+
+ async def alist_fine_tuning_jobs(
+ self,
+ openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI],
+ after: Optional[str] = None,
+ limit: Optional[int] = None,
+ ):
+ response = await openai_client.fine_tuning.jobs.list(after=after, limit=limit) # type: ignore
+ return response
+
+ def list_fine_tuning_jobs(
+ self,
+ _is_async: bool,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ api_version: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[
+ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
+ ] = None,
+ after: Optional[str] = None,
+ limit: Optional[int] = None,
+ ):
+ openai_client: Optional[
+ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
+ ] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ api_version=api_version,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, (AsyncOpenAI, AsyncAzureOpenAI)):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.alist_fine_tuning_jobs( # type: ignore
+ after=after,
+ limit=limit,
+ openai_client=openai_client,
+ )
+ verbose_logger.debug("list fine tuning job, after= %s, limit= %s", after, limit)
+ response = openai_client.fine_tuning.jobs.list(after=after, limit=limit) # type: ignore
+ return response
+
+ async def aretrieve_fine_tuning_job(
+ self,
+ fine_tuning_job_id: str,
+ openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI],
+ ) -> LiteLLMFineTuningJob:
+ response = await openai_client.fine_tuning.jobs.retrieve(
+ fine_tuning_job_id=fine_tuning_job_id
+ )
+ return LiteLLMFineTuningJob(**response.model_dump())
+
+ def retrieve_fine_tuning_job(
+ self,
+ _is_async: bool,
+ fine_tuning_job_id: str,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ api_version: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[
+ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
+ ] = None,
+ ) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]:
+ openai_client: Optional[
+ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI]
+ ] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ api_version=api_version,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, AsyncOpenAI):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.aretrieve_fine_tuning_job( # type: ignore
+ fine_tuning_job_id=fine_tuning_job_id,
+ openai_client=openai_client,
+ )
+ verbose_logger.debug("retrieving fine tuning job, id= %s", fine_tuning_job_id)
+ response = cast(OpenAI, openai_client).fine_tuning.jobs.retrieve(
+ fine_tuning_job_id=fine_tuning_job_id
+ )
+ return LiteLLMFineTuningJob(**response.model_dump())
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_edit/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_edit/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..5d933b8186d6f7cf563897865e9dcc06315088d6
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_edit/__init__.py
@@ -0,0 +1,29 @@
+from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
+
+from .dalle2_transformation import DallE2ImageEditConfig
+from .transformation import OpenAIImageEditConfig
+
+__all__ = [
+ "OpenAIImageEditConfig",
+ "DallE2ImageEditConfig",
+ "get_openai_image_edit_config",
+]
+
+
+def get_openai_image_edit_config(model: str) -> BaseImageEditConfig:
+ """
+ Get the appropriate OpenAI image edit config based on the model.
+
+ Args:
+ model: The model name (e.g., "dall-e-2", "gpt-image-1")
+
+ Returns:
+ The appropriate config instance for the model
+ """
+ model_normalized = model.lower().replace("-", "").replace("_", "")
+
+ if model_normalized == "dalle2":
+ return DallE2ImageEditConfig()
+ else:
+ # Default to standard OpenAI config for gpt-image-1 and other models
+ return OpenAIImageEditConfig()
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_edit/dalle2_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_edit/dalle2_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..04995ce95143ddd9b1202cc41966ff131b9aedad
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_edit/dalle2_transformation.py
@@ -0,0 +1,104 @@
+from io import BufferedReader
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
+
+from httpx._types import RequestFiles
+
+import litellm
+from litellm.images.utils import ImageEditRequestUtils
+from litellm.types.images.main import ImageEditRequestParams
+from litellm.types.llms.openai import FileTypes
+from litellm.types.router import GenericLiteLLMParams
+
+from .transformation import OpenAIImageEditConfig
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+ LiteLLMLoggingObj = Any
+
+
+class DallE2ImageEditConfig(OpenAIImageEditConfig):
+ """
+ DALL-E-2 specific configuration for image edit API.
+
+ DALL-E-2 only supports editing a single image (not an array).
+ Uses "image" field name instead of "image[]".
+ """
+
+ def transform_image_edit_request(
+ self,
+ model: str,
+ prompt: Optional[str],
+ image: Optional[FileTypes],
+ image_edit_optional_request_params: Dict,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[Dict, RequestFiles]:
+ """
+ Transform image edit request for DALL-E-2.
+
+ DALL-E-2 only accepts a single image with field name "image" (not "image[]").
+ """
+ request_params = {
+ "model": model,
+ **image_edit_optional_request_params,
+ }
+ if image is not None:
+ request_params["image"] = image
+ if prompt is not None:
+ request_params["prompt"] = prompt
+
+ request = ImageEditRequestParams(**request_params)
+ request_dict = cast(Dict, request)
+
+ #########################################################
+ # Separate images and masks as `files` and send other parameters as `data`
+ #########################################################
+ _image_list = request_dict.get("image")
+ _mask = request_dict.get("mask")
+ data_without_files = {
+ k: v for k, v in request_dict.items() if k not in ["image", "mask"]
+ }
+ files_list: List[Tuple[str, Any]] = []
+
+ # Handle image parameter - DALL-E-2 only supports single image
+ if _image_list is not None:
+ image_list = (
+ [_image_list] if not isinstance(_image_list, list) else _image_list
+ )
+
+ # Validate only one image is provided
+ if len(image_list) > 1:
+ raise litellm.BadRequestError(
+ message="DALL-E-2 only supports editing a single image. Please provide one image.",
+ model=model,
+ llm_provider="openai",
+ )
+
+ # Use "image" field name (singular) for DALL-E-2
+ for _image in image_list:
+ if _image is not None:
+ self._add_image_to_files(
+ files_list=files_list,
+ image=_image,
+ field_name="image",
+ )
+
+ # Handle mask parameter if provided
+ if _mask is not None:
+ # Handle case where mask can be a list (extract first mask)
+ if isinstance(_mask, list):
+ _mask = _mask[0] if _mask else None
+
+ if _mask is not None:
+ mask_content_type: str = ImageEditRequestUtils.get_image_content_type(
+ _mask
+ )
+ if isinstance(_mask, BufferedReader):
+ files_list.append(("mask", (_mask.name, _mask, mask_content_type)))
+ else:
+ files_list.append(("mask", ("mask.png", _mask, mask_content_type)))
+
+ return data_without_files, files_list
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_edit/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_edit/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..6917e8d79900baf87437cc212f2b3c3d0cc5b147
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_edit/transformation.py
@@ -0,0 +1,202 @@
+from io import BufferedReader
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
+
+import httpx
+from httpx._types import RequestFiles
+
+import litellm
+from litellm.images.utils import ImageEditRequestUtils
+from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.images.main import (
+ ImageEditOptionalRequestParams,
+ ImageEditRequestParams,
+)
+from litellm.types.llms.openai import FileTypes
+from litellm.types.router import GenericLiteLLMParams
+from litellm.utils import ImageResponse
+
+from ..common_utils import OpenAIError
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+ LiteLLMLoggingObj = Any
+
+
+class OpenAIImageEditConfig(BaseImageEditConfig):
+ """
+ Base configuration for OpenAI image edit API.
+ Used for models like gpt-image-1 that support multiple images.
+ """
+
+ def get_supported_openai_params(self, model: str) -> list:
+ """
+ All OpenAI Image Edits params are supported
+ """
+ return [
+ "image",
+ "prompt",
+ "background",
+ "input_fidelity",
+ "mask",
+ "model",
+ "n",
+ "quality",
+ "response_format",
+ "size",
+ "user",
+ "extra_headers",
+ "extra_query",
+ "extra_body",
+ "timeout",
+ ]
+
+ def map_openai_params(
+ self,
+ image_edit_optional_params: ImageEditOptionalRequestParams,
+ model: str,
+ drop_params: bool,
+ ) -> Dict:
+ """No mapping applied since inputs are in OpenAI spec already"""
+ return dict(image_edit_optional_params)
+
+ def _add_image_to_files(
+ self,
+ files_list: List[Tuple[str, Any]],
+ image: Any,
+ field_name: str,
+ ) -> None:
+ """Add an image to the files list with appropriate content type"""
+ image_content_type = ImageEditRequestUtils.get_image_content_type(image)
+
+ if isinstance(image, BufferedReader):
+ files_list.append((field_name, (image.name, image, image_content_type)))
+ else:
+ files_list.append((field_name, ("image.png", image, image_content_type)))
+
+ def transform_image_edit_request(
+ self,
+ model: str,
+ prompt: Optional[str],
+ image: Optional[FileTypes],
+ image_edit_optional_request_params: Dict,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[Dict, RequestFiles]:
+ """
+ Transform image edit request to OpenAI API format.
+
+ Handles multipart/form-data for images. Uses "image[]" field name
+ to support multiple images (e.g., for gpt-image-1).
+ """
+ # Build request params, only including non-None values
+ request_params = {
+ "model": model,
+ **image_edit_optional_request_params,
+ }
+ if image is not None:
+ request_params["image"] = image
+ if prompt is not None:
+ request_params["prompt"] = prompt
+
+ request = ImageEditRequestParams(**request_params)
+ request_dict = cast(Dict, request)
+
+ #########################################################
+ # Separate images and masks as `files` and send other parameters as `data`
+ #########################################################
+ _image_list = request_dict.get("image")
+ _mask = request_dict.get("mask")
+ data_without_files = {
+ k: v for k, v in request_dict.items() if k not in ["image", "mask"]
+ }
+ files_list: List[Tuple[str, Any]] = []
+
+ # Handle image parameter
+ if _image_list is not None:
+ image_list = (
+ [_image_list] if not isinstance(_image_list, list) else _image_list
+ )
+
+ for _image in image_list:
+ if _image is not None:
+ self._add_image_to_files(
+ files_list=files_list,
+ image=_image,
+ field_name="image[]",
+ )
+ # Handle mask parameter if provided
+ if _mask is not None:
+ # Handle case where mask can be a list (extract first mask)
+ if isinstance(_mask, list):
+ _mask = _mask[0] if _mask else None
+
+ if _mask is not None:
+ mask_content_type: str = ImageEditRequestUtils.get_image_content_type(
+ _mask
+ )
+ if isinstance(_mask, BufferedReader):
+ files_list.append(("mask", (_mask.name, _mask, mask_content_type)))
+ else:
+ files_list.append(("mask", ("mask.png", _mask, mask_content_type)))
+
+ return data_without_files, files_list
+
+ def transform_image_edit_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ImageResponse:
+ """No transform applied since outputs are in OpenAI spec already"""
+ try:
+ raw_response_json = raw_response.json()
+ except Exception:
+ raise OpenAIError(
+ message=raw_response.text, status_code=raw_response.status_code
+ )
+ return ImageResponse(**raw_response_json)
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ api_key: Optional[str] = None,
+ ) -> dict:
+ api_key = (
+ api_key
+ or litellm.api_key
+ or litellm.openai_key
+ or get_secret_str("OPENAI_API_KEY")
+ )
+ headers.update(
+ {
+ "Authorization": f"Bearer {api_key}",
+ }
+ )
+ return headers
+
+ def get_complete_url(
+ self,
+ model: str,
+ api_base: Optional[str],
+ litellm_params: dict,
+ ) -> str:
+ """
+ Get the endpoint for OpenAI responses API
+ """
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("OPENAI_BASE_URL")
+ or get_secret_str("OPENAI_API_BASE")
+ or "https://api.openai.com/v1"
+ )
+
+ # Remove trailing slashes
+ api_base = api_base.rstrip("/")
+
+ return f"{api_base}/images/edits"
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e20c80f20bbf81adaa7f4f9a02c90f1d9137a0a2
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/__init__.py
@@ -0,0 +1,28 @@
+from litellm.llms.base_llm.image_generation.transformation import (
+ BaseImageGenerationConfig,
+)
+
+from .dall_e_2_transformation import DallE2ImageGenerationConfig
+from .dall_e_3_transformation import DallE3ImageGenerationConfig
+from .gpt_transformation import GPTImageGenerationConfig
+from .guardrail_translation import (
+ OpenAIImageGenerationHandler,
+ guardrail_translation_mappings,
+)
+
+__all__ = [
+ "DallE2ImageGenerationConfig",
+ "DallE3ImageGenerationConfig",
+ "GPTImageGenerationConfig",
+ "OpenAIImageGenerationHandler",
+ "guardrail_translation_mappings",
+]
+
+
+def get_openai_image_generation_config(model: str) -> BaseImageGenerationConfig:
+ if model.startswith("dall-e-2") or model == "": # empty model is dall-e-2
+ return DallE2ImageGenerationConfig()
+ elif model.startswith("dall-e-3"):
+ return DallE3ImageGenerationConfig()
+ else:
+ return GPTImageGenerationConfig()
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/cost_calculator.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/cost_calculator.py
new file mode 100644
index 0000000000000000000000000000000000000000..8bca75172faeb5f6bbc1f24678849a3c57104c2e
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/cost_calculator.py
@@ -0,0 +1,69 @@
+"""
+Cost calculator for OpenAI image generation models (gpt-image-1, gpt-image-1-mini)
+
+These models use token-based pricing instead of pixel-based pricing like DALL-E.
+"""
+
+from typing import Optional
+
+from litellm import verbose_logger
+from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
+from litellm.types.utils import ImageResponse, Usage
+
+
+def cost_calculator(
+ model: str,
+ image_response: ImageResponse,
+ custom_llm_provider: Optional[str] = None,
+) -> float:
+ """
+ Calculate cost for OpenAI gpt-image-1 and gpt-image-1-mini models.
+
+ Uses the same usage format as Responses API, so we reuse the helper
+ to transform to chat completion format and use generic_cost_per_token.
+
+ Args:
+ model: The model name (e.g., "gpt-image-1", "gpt-image-1-mini")
+ image_response: The ImageResponse containing usage data
+ custom_llm_provider: Optional provider name
+
+ Returns:
+ float: Total cost in USD
+ """
+ usage = getattr(image_response, "usage", None)
+
+ if usage is None:
+ verbose_logger.debug(
+ f"No usage data available for {model}, cannot calculate token-based cost"
+ )
+ return 0.0
+
+ # If usage is already a Usage object with completion_tokens_details set,
+ # use it directly (it was already transformed in convert_to_image_response)
+ if isinstance(usage, Usage) and usage.completion_tokens_details is not None:
+ chat_usage = usage
+ else:
+ # Transform ImageUsage to Usage using the existing helper
+ # ImageUsage has the same format as ResponseAPIUsage
+ from litellm.responses.utils import ResponseAPILoggingUtils
+
+ chat_usage = (
+ ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
+ )
+
+ # Use generic_cost_per_token for cost calculation
+ prompt_cost, completion_cost = generic_cost_per_token(
+ model=model,
+ usage=chat_usage,
+ custom_llm_provider=custom_llm_provider or "openai",
+ )
+
+ total_cost = prompt_cost + completion_cost
+
+ verbose_logger.debug(
+ f"OpenAI gpt-image cost calculation for {model}: "
+ f"prompt_cost=${prompt_cost:.6f}, completion_cost=${completion_cost:.6f}, "
+ f"total=${total_cost:.6f}"
+ )
+
+ return total_cost
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/dall_e_2_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/dall_e_2_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..22c2349a8370246fac7da619f4509c300ccfb0a5
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/dall_e_2_transformation.py
@@ -0,0 +1,87 @@
+from typing import TYPE_CHECKING, Any, List, Optional
+
+import httpx
+
+from litellm.llms.base_llm.image_generation.transformation import (
+ BaseImageGenerationConfig,
+)
+from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
+from litellm.types.utils import ImageResponse
+from litellm.utils import convert_to_model_response_object
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
+
+
+class DallE2ImageGenerationConfig(BaseImageGenerationConfig):
+ """
+ OpenAI dall-e-2 image generation config
+ """
+
+ def get_supported_openai_params(
+ self, model: str
+ ) -> List[OpenAIImageGenerationOptionalParams]:
+ return ["n", "response_format", "quality", "size", "user"]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ supported_params = self.get_supported_openai_params(model)
+ for k in non_default_params.keys():
+ if k not in optional_params.keys():
+ if k in supported_params:
+ optional_params[k] = non_default_params[k]
+ elif drop_params:
+ pass
+ else:
+ raise ValueError(
+ f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
+ )
+
+ return optional_params
+
+ def transform_image_generation_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ model_response: ImageResponse,
+ logging_obj: "LiteLLMLoggingObj",
+ request_data: dict,
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: Any,
+ api_key: Optional[str] = None,
+ json_mode: Optional[bool] = None,
+ ) -> ImageResponse:
+ response = raw_response.json()
+
+ stringified_response = response
+ ## LOGGING
+ logging_obj.post_call(
+ input=request_data.get("prompt", ""),
+ api_key=api_key,
+ additional_args={"complete_input_dict": request_data},
+ original_response=stringified_response,
+ )
+ image_response: ImageResponse = convert_to_model_response_object( # type: ignore
+ response_object=stringified_response,
+ model_response_object=model_response,
+ response_type="image_generation",
+ )
+
+ # set optional params
+ image_response.size = optional_params.get(
+ "size", "1024x1024"
+ ) # default is always 1024x1024
+ image_response.quality = optional_params.get(
+ "quality", "standard"
+ ) # always standard for dall-e-2
+ image_response.output_format = optional_params.get(
+ "output_format", "png"
+ ) # always png for dall-e-2
+
+ return image_response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/dall_e_3_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/dall_e_3_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e2bdabc3a1449bfa8ca4d84871ea4a27173f5d6
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/dall_e_3_transformation.py
@@ -0,0 +1,87 @@
+from typing import TYPE_CHECKING, Any, List, Optional
+
+import httpx
+
+from litellm.llms.base_llm.image_generation.transformation import (
+ BaseImageGenerationConfig,
+)
+from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
+from litellm.types.utils import ImageResponse
+from litellm.utils import convert_to_model_response_object
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
+
+
+class DallE3ImageGenerationConfig(BaseImageGenerationConfig):
+ """
+ OpenAI dall-e-3 image generation config
+ """
+
+ def get_supported_openai_params(
+ self, model: str
+ ) -> List[OpenAIImageGenerationOptionalParams]:
+ return ["n", "response_format", "quality", "size", "user", "style"]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ supported_params = self.get_supported_openai_params(model)
+ for k in non_default_params.keys():
+ if k not in optional_params.keys():
+ if k in supported_params:
+ optional_params[k] = non_default_params[k]
+ elif drop_params:
+ pass
+ else:
+ raise ValueError(
+ f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
+ )
+
+ return optional_params
+
+ def transform_image_generation_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ model_response: ImageResponse,
+ logging_obj: "LiteLLMLoggingObj",
+ request_data: dict,
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: Any,
+ api_key: Optional[str] = None,
+ json_mode: Optional[bool] = None,
+ ) -> ImageResponse:
+ response = raw_response.json()
+
+ stringified_response = response
+ ## LOGGING
+ logging_obj.post_call(
+ input=request_data.get("prompt", ""),
+ api_key=api_key,
+ additional_args={"complete_input_dict": request_data},
+ original_response=stringified_response,
+ )
+ image_response: ImageResponse = convert_to_model_response_object( # type: ignore
+ response_object=stringified_response,
+ model_response_object=model_response,
+ response_type="image_generation",
+ )
+
+ # set optional params
+ image_response.size = optional_params.get(
+ "size", "1024x1024"
+ ) # default is always 1024x1024
+ image_response.quality = optional_params.get(
+ "quality", "hd"
+ ) # always hd for dall-e-3
+ image_response.output_format = optional_params.get(
+ "output_format", "png"
+ ) # always png for dall-e-3
+
+ return image_response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/gpt_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/gpt_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..c106d7f17b63d8bc88fa6d0c05f1d381466dc02c
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/gpt_transformation.py
@@ -0,0 +1,96 @@
+from typing import TYPE_CHECKING, Any, List, Optional
+
+import httpx
+
+from litellm.llms.base_llm.image_generation.transformation import (
+ BaseImageGenerationConfig,
+)
+from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
+from litellm.types.utils import ImageResponse
+from litellm.utils import convert_to_model_response_object
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
+
+
+class GPTImageGenerationConfig(BaseImageGenerationConfig):
+ """
+ OpenAI gpt-image-1 image generation config
+ """
+
+ def get_supported_openai_params(
+ self, model: str
+ ) -> List[OpenAIImageGenerationOptionalParams]:
+ return [
+ "background",
+ "moderation",
+ "n",
+ "output_compression",
+ "output_format",
+ "quality",
+ "size",
+ "user",
+ ]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ supported_params = self.get_supported_openai_params(model)
+ for k in non_default_params.keys():
+ if k not in optional_params.keys():
+ if k in supported_params:
+ optional_params[k] = non_default_params[k]
+ elif drop_params:
+ pass
+ else:
+ raise ValueError(
+ f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
+ )
+
+ return optional_params
+
+ def transform_image_generation_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ model_response: ImageResponse,
+ logging_obj: "LiteLLMLoggingObj",
+ request_data: dict,
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: Any,
+ api_key: Optional[str] = None,
+ json_mode: Optional[bool] = None,
+ ) -> ImageResponse:
+ response = raw_response.json()
+
+ stringified_response = response
+ ## LOGGING
+ logging_obj.post_call(
+ input=request_data.get("prompt", ""),
+ api_key=api_key,
+ additional_args={"complete_input_dict": request_data},
+ original_response=stringified_response,
+ )
+ image_response: ImageResponse = convert_to_model_response_object( # type: ignore
+ response_object=stringified_response,
+ model_response_object=model_response,
+ response_type="image_generation",
+ )
+
+ # set optional params
+ image_response.size = optional_params.get(
+ "size", "1024x1024"
+ ) # default is always 1024x1024
+ image_response.quality = optional_params.get(
+ "quality", "high"
+ ) # always hd for dall-e-3
+ image_response.output_format = optional_params.get(
+ "response_format", "png"
+ ) # always png for dall-e-3
+
+ return image_response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/guardrail_translation/README.md b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/guardrail_translation/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..fcbd2d154dedc13eb86c54f350ed343b2a4341f1
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/guardrail_translation/README.md
@@ -0,0 +1,106 @@
+# OpenAI Image Generation Guardrail Translation Handler
+
+Handler for processing OpenAI's image generation endpoint with guardrails.
+
+## Overview
+
+This handler processes image generation requests by:
+1. Extracting the text prompt from the request
+2. Applying guardrails to the prompt text
+3. Updating the request with the guardrailed prompt
+
+## Data Format
+
+### Input Format
+
+```json
+{
+ "model": "dall-e-3",
+ "prompt": "A cute baby sea otter",
+ "n": 1,
+ "size": "1024x1024",
+ "quality": "standard"
+}
+```
+
+### Output Format
+
+```json
+{
+ "created": 1589478378,
+ "data": [
+ {
+ "url": "https://...",
+ "revised_prompt": "A cute baby sea otter..."
+ }
+ ]
+}
+```
+
+## Usage
+
+The handler is automatically discovered and applied when guardrails are used with the image generation endpoint.
+
+### Example: Using Guardrails with Image Generation
+
+```bash
+curl -X POST 'http://localhost:4000/v1/images/generations' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer your-api-key' \
+-d '{
+ "model": "dall-e-3",
+ "prompt": "A cute baby sea otter wearing a hat",
+ "guardrails": ["content_moderation"],
+ "size": "1024x1024"
+}'
+```
+
+The guardrail will be applied to the prompt text before the image generation request is sent to the provider.
+
+### Example: PII Masking in Prompts
+
+```bash
+curl -X POST 'http://localhost:4000/v1/images/generations' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer your-api-key' \
+-d '{
+ "model": "dall-e-3",
+ "prompt": "Generate an image of John Doe at john@example.com",
+ "guardrails": ["mask_pii"],
+ "metadata": {
+ "guardrails": ["mask_pii"]
+ }
+}'
+```
+
+## Implementation Details
+
+### Input Processing
+
+- **Field**: `prompt` (string)
+- **Processing**: Applies guardrail to prompt text
+- **Result**: Updated prompt in request
+
+### Output Processing
+
+- **Processing**: Not applicable (images don't contain text to guardrail)
+- **Result**: Response returned unchanged
+
+## Extension
+
+Override these methods to customize behavior:
+
+- `process_input_messages()`: Customize how the prompt is processed
+- `process_output_response()`: Add custom processing for image metadata if needed
+
+## Supported Call Types
+
+- `CallTypes.image_generation` - Synchronous image generation
+- `CallTypes.aimage_generation` - Asynchronous image generation
+
+## Notes
+
+- The handler only processes the `prompt` parameter
+- Output processing is a no-op since images don't contain text
+- Both sync and async call types use the same handler
+
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/guardrail_translation/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/guardrail_translation/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..1fba2a36927cbed8520d35b17f04521364e2c1d4
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/guardrail_translation/__init__.py
@@ -0,0 +1,13 @@
+"""OpenAI Image Generation handler for Unified Guardrails."""
+
+from litellm.llms.openai.image_generation.guardrail_translation.handler import (
+ OpenAIImageGenerationHandler,
+)
+from litellm.types.utils import CallTypes
+
+guardrail_translation_mappings = {
+ CallTypes.image_generation: OpenAIImageGenerationHandler,
+ CallTypes.aimage_generation: OpenAIImageGenerationHandler,
+}
+
+__all__ = ["guardrail_translation_mappings", "OpenAIImageGenerationHandler"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/guardrail_translation/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/guardrail_translation/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..e6340ba4705a9519d790006545a1e2609061d886
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_generation/guardrail_translation/handler.py
@@ -0,0 +1,110 @@
+"""
+OpenAI Image Generation Handler for Unified Guardrails
+
+This module provides guardrail translation support for OpenAI's image generation endpoint.
+The handler processes the 'prompt' parameter for guardrails.
+"""
+
+from typing import TYPE_CHECKING, Any, Optional
+
+from litellm._logging import verbose_proxy_logger
+from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
+from litellm.types.utils import GenericGuardrailAPIInputs
+
+if TYPE_CHECKING:
+ from litellm.integrations.custom_guardrail import CustomGuardrail
+ from litellm.utils import ImageResponse
+
+
+class OpenAIImageGenerationHandler(BaseTranslation):
+ """
+ Handler for processing OpenAI image generation requests with guardrails.
+
+ This class provides methods to:
+ 1. Process input prompt (pre-call hook)
+ 2. Process output response (post-call hook) - typically not needed for images
+
+ The handler specifically processes the 'prompt' parameter which contains
+ the text description for image generation.
+ """
+
+ async def process_input_messages(
+ self,
+ data: dict,
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process input prompt by applying guardrails to text content.
+
+ Args:
+ data: Request data dictionary containing 'prompt' parameter
+ guardrail_to_apply: The guardrail instance to apply
+
+ Returns:
+ Modified data with guardrails applied to prompt
+ """
+ prompt = data.get("prompt")
+ if prompt is None:
+ verbose_proxy_logger.debug(
+ "OpenAI Image Generation: No prompt found in request data"
+ )
+ return data
+
+ # Apply guardrail to the prompt
+ if isinstance(prompt, str):
+ inputs = GenericGuardrailAPIInputs(texts=[prompt])
+ # Include model information if available
+ model = data.get("model")
+ if model:
+ inputs["model"] = model
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=data,
+ input_type="request",
+ logging_obj=litellm_logging_obj,
+ )
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+ data["prompt"] = guardrailed_texts[0] if guardrailed_texts else prompt
+
+ verbose_proxy_logger.debug(
+ "OpenAI Image Generation: Applied guardrail to prompt. "
+ "Original length: %d, New length: %d",
+ len(prompt),
+ len(data["prompt"]),
+ )
+ else:
+ verbose_proxy_logger.debug(
+ "OpenAI Image Generation: Unexpected prompt type: %s. Expected string.",
+ type(prompt),
+ )
+
+ return data
+
+ async def process_output_response(
+ self,
+ response: "ImageResponse",
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ user_api_key_dict: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process output response - typically not needed for image generation.
+
+ Image responses don't contain text to apply guardrails to, so this
+ method returns the response unchanged. This is provided for completeness
+ and can be overridden if needed for custom image metadata processing.
+
+ Args:
+ response: Image generation response object
+ guardrail_to_apply: The guardrail instance to apply
+ litellm_logging_obj: Optional logging object (unused)
+ user_api_key_dict: User API key metadata (unused)
+
+ Returns:
+ Unmodified response (images don't need text guardrails)
+ """
+ verbose_proxy_logger.debug(
+ "OpenAI Image Generation: Output processing not needed for image responses"
+ )
+ return response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_variations/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_variations/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..8b96fb6ef7a7a764adffc33a83d390b4d8a0cacf
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_variations/handler.py
@@ -0,0 +1,244 @@
+"""
+OpenAI Image Variations Handler
+"""
+
+from typing import Callable, Optional
+
+import httpx
+from openai import AsyncOpenAI, OpenAI
+
+import litellm
+from litellm.types.utils import FileTypes, ImageResponse, LlmProviders
+from litellm.utils import ProviderConfigManager
+
+from ...base_llm.image_variations.transformation import BaseImageVariationConfig
+from ...custom_httpx.llm_http_handler import LiteLLMLoggingObj
+from ..common_utils import OpenAIError
+
+
+class OpenAIImageVariationsHandler:
+ def get_sync_client(
+ self,
+ client: Optional[OpenAI],
+ init_client_params: dict,
+ ):
+ if client is None:
+ openai_client = OpenAI(
+ **init_client_params,
+ )
+ else:
+ openai_client = client
+ return openai_client
+
+ def get_async_client(
+ self, client: Optional[AsyncOpenAI], init_client_params: dict
+ ) -> AsyncOpenAI:
+ if client is None:
+ openai_client = AsyncOpenAI(
+ **init_client_params,
+ )
+ else:
+ openai_client = client
+ return openai_client
+
+ async def async_image_variations(
+ self,
+ api_key: str,
+ api_base: str,
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI],
+ data: dict,
+ headers: dict,
+ model: Optional[str],
+ timeout: Optional[float],
+ max_retries: int,
+ logging_obj: LiteLLMLoggingObj,
+ model_response: ImageResponse,
+ optional_params: dict,
+ litellm_params: dict,
+ image: FileTypes,
+ provider_config: BaseImageVariationConfig,
+ ) -> ImageResponse:
+ try:
+ init_client_params = {
+ "api_key": api_key,
+ "base_url": api_base,
+ "http_client": litellm.client_session,
+ "timeout": timeout,
+ "max_retries": max_retries, # type: ignore
+ "organization": organization,
+ }
+
+ client = self.get_async_client(
+ client=client, init_client_params=init_client_params
+ )
+
+ raw_response = await client.images.with_raw_response.create_variation(**data) # type: ignore
+ response = raw_response.parse()
+ response_json = response.model_dump()
+
+ ## LOGGING
+ logging_obj.post_call(
+ api_key=api_key,
+ original_response=response_json,
+ additional_args={
+ "headers": headers,
+ "api_base": api_base,
+ },
+ )
+
+ ## RESPONSE OBJECT
+ return provider_config.transform_response_image_variation(
+ model=model,
+ model_response=ImageResponse(**response_json),
+ raw_response=httpx.Response(
+ status_code=200,
+ request=httpx.Request(
+ method="GET", url="https://litellm.ai"
+ ), # mock request object
+ ),
+ logging_obj=logging_obj,
+ request_data=data,
+ image=image,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ encoding=None,
+ api_key=api_key,
+ )
+ except Exception as e:
+ status_code = getattr(e, "status_code", 500)
+ error_headers = getattr(e, "headers", None)
+ error_text = getattr(e, "text", str(e))
+ error_response = getattr(e, "response", None)
+ if error_headers is None and error_response:
+ error_headers = getattr(error_response, "headers", None)
+ raise OpenAIError(
+ status_code=status_code, message=error_text, headers=error_headers
+ )
+
+ def image_variations(
+ self,
+ model_response: ImageResponse,
+ api_key: str,
+ api_base: str,
+ model: Optional[str],
+ image: FileTypes,
+ timeout: Optional[float],
+ custom_llm_provider: str,
+ logging_obj: LiteLLMLoggingObj,
+ optional_params: dict,
+ litellm_params: dict,
+ print_verbose: Optional[Callable] = None,
+ logger_fn=None,
+ client=None,
+ organization: Optional[str] = None,
+ headers: Optional[dict] = None,
+ ) -> ImageResponse:
+ try:
+ provider_config = ProviderConfigManager.get_provider_image_variation_config(
+ model=model or "", # openai defaults to dall-e-2
+ provider=LlmProviders.OPENAI,
+ )
+
+ if provider_config is None:
+ raise ValueError(
+ f"image variation provider not found: {custom_llm_provider}."
+ )
+
+ max_retries = optional_params.pop("max_retries", 2)
+
+ data = provider_config.transform_request_image_variation(
+ model=model,
+ image=image,
+ optional_params=optional_params,
+ headers=headers or {},
+ )
+ json_data = data.get("data")
+ if not json_data:
+ raise ValueError(
+ f"data field is required, for openai image variations. Got={data}"
+ )
+ ## LOGGING
+ logging_obj.pre_call(
+ input="",
+ api_key=api_key,
+ additional_args={
+ "headers": headers,
+ "api_base": api_base,
+ "complete_input_dict": data,
+ },
+ )
+ if litellm_params.get("async_call", False):
+ return self.async_image_variations(
+ api_base=api_base,
+ data=json_data,
+ headers=headers or {},
+ model_response=model_response,
+ api_key=api_key,
+ logging_obj=logging_obj,
+ model=model,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ provider_config=provider_config,
+ image=image,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ ) # type: ignore
+
+ init_client_params = {
+ "api_key": api_key,
+ "base_url": api_base,
+ "http_client": litellm.client_session,
+ "timeout": timeout,
+ "max_retries": max_retries, # type: ignore
+ "organization": organization,
+ }
+
+ client = self.get_sync_client(
+ client=client, init_client_params=init_client_params
+ )
+
+ raw_response = client.images.with_raw_response.create_variation(**json_data) # type: ignore
+ response = raw_response.parse()
+ response_json = response.model_dump()
+
+ ## LOGGING
+ logging_obj.post_call(
+ api_key=api_key,
+ original_response=response_json,
+ additional_args={
+ "headers": headers,
+ "api_base": api_base,
+ },
+ )
+
+ ## RESPONSE OBJECT
+ return provider_config.transform_response_image_variation(
+ model=model,
+ model_response=ImageResponse(**response_json),
+ raw_response=httpx.Response(
+ status_code=200,
+ request=httpx.Request(
+ method="GET", url="https://litellm.ai"
+ ), # mock request object
+ ),
+ logging_obj=logging_obj,
+ request_data=json_data,
+ image=image,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ encoding=None,
+ api_key=api_key,
+ )
+ except Exception as e:
+ status_code = getattr(e, "status_code", 500)
+ error_headers = getattr(e, "headers", None)
+ error_text = getattr(e, "text", str(e))
+ error_response = getattr(e, "response", None)
+ if error_headers is None and error_response:
+ error_headers = getattr(error_response, "headers", None)
+ raise OpenAIError(
+ status_code=status_code, message=error_text, headers=error_headers
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_variations/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_variations/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..96d1a302761f24e654d779ee31e36d34dc38507c
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/image_variations/transformation.py
@@ -0,0 +1,82 @@
+from typing import Any, List, Optional, Union
+
+from aiohttp import ClientResponse
+from httpx import Headers, Response
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.image_variations.transformation import LiteLLMLoggingObj
+from litellm.types.llms.openai import OpenAIImageVariationOptionalParams
+from litellm.types.utils import FileTypes, HttpHandlerRequestFields, ImageResponse
+
+from ...base_llm.image_variations.transformation import BaseImageVariationConfig
+from ..common_utils import OpenAIError
+
+
+class OpenAIImageVariationConfig(BaseImageVariationConfig):
+ def get_supported_openai_params(
+ self, model: str
+ ) -> List[OpenAIImageVariationOptionalParams]:
+ return ["n", "size", "response_format", "user"]
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ optional_params.update(non_default_params)
+ return optional_params
+
+ def transform_request_image_variation(
+ self,
+ model: Optional[str],
+ image: FileTypes,
+ optional_params: dict,
+ headers: dict,
+ ) -> HttpHandlerRequestFields:
+ return {
+ "data": {
+ "image": image,
+ **optional_params,
+ }
+ }
+
+ async def async_transform_response_image_variation(
+ self,
+ model: Optional[str],
+ raw_response: ClientResponse,
+ model_response: ImageResponse,
+ logging_obj: LiteLLMLoggingObj,
+ request_data: dict,
+ image: FileTypes,
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: Any,
+ api_key: Optional[str] = None,
+ ) -> ImageResponse:
+ return model_response
+
+ def transform_response_image_variation(
+ self,
+ model: Optional[str],
+ raw_response: Response,
+ model_response: ImageResponse,
+ logging_obj: LiteLLMLoggingObj,
+ request_data: dict,
+ image: FileTypes,
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: Any,
+ api_key: Optional[str] = None,
+ ) -> ImageResponse:
+ return model_response
+
+ def get_error_class(
+ self, error_message: str, status_code: int, headers: Union[dict, Headers]
+ ) -> BaseLLMException:
+ return OpenAIError(
+ status_code=status_code,
+ message=error_message,
+ headers=headers,
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/openai.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/openai.py
new file mode 100644
index 0000000000000000000000000000000000000000..be542677480eaaa31701834739fbb6efc537dd38
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/openai.py
@@ -0,0 +1,3048 @@
+import time
+import types
+from typing import (
+ TYPE_CHECKING,
+ Any,
+ AsyncIterator,
+ Callable,
+ Coroutine,
+ Iterable,
+ Iterator,
+ List,
+ Literal,
+ Optional,
+ Union,
+ cast,
+)
+from urllib.parse import urlparse
+
+import httpx
+
+if TYPE_CHECKING:
+ from aiohttp import ClientSession
+
+import openai
+from openai import AsyncOpenAI, OpenAI
+from openai.types.beta.assistant_deleted import AssistantDeleted
+from openai.types.file_deleted import FileDeleted
+from pydantic import BaseModel
+from typing_extensions import overload
+
+import litellm
+from litellm import LlmProviders
+from litellm._logging import verbose_logger
+from litellm.constants import DEFAULT_MAX_RETRIES
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
+from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
+from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
+from litellm.llms.bedrock.chat.invoke_handler import MockResponseIterator
+from litellm.types.utils import (
+ EmbeddingResponse,
+ ImageResponse,
+ LiteLLMBatch,
+ ModelResponse,
+ ModelResponseStream,
+)
+from litellm.utils import (
+ CustomStreamWrapper,
+ ProviderConfigManager,
+ convert_to_model_response_object,
+)
+
+from ...types.llms.openai import *
+from ..base import BaseLLM
+from .chat.gpt_5_transformation import OpenAIGPT5Config
+from .chat.o_series_transformation import OpenAIOSeriesConfig
+from .common_utils import (
+ BaseOpenAILLM,
+ OpenAIError,
+ drop_params_from_unprocessable_entity_error,
+)
+
+openaiOSeriesConfig = OpenAIOSeriesConfig()
+openAIGPT5Config = OpenAIGPT5Config()
+
+
+class MistralEmbeddingConfig:
+ """
+ Reference: https://docs.mistral.ai/api/#operation/createEmbedding
+ """
+
+ def __init__(
+ self,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return {
+ k: v
+ for k, v in cls.__dict__.items()
+ if not k.startswith("__")
+ and not isinstance(
+ v,
+ (
+ types.FunctionType,
+ types.BuiltinFunctionType,
+ classmethod,
+ staticmethod,
+ ),
+ )
+ and v is not None
+ }
+
+ def get_supported_openai_params(self):
+ return [
+ "encoding_format",
+ ]
+
+ def map_openai_params(self, non_default_params: dict, optional_params: dict):
+ for param, value in non_default_params.items():
+ if param == "encoding_format":
+ optional_params["encoding_format"] = value
+ return optional_params
+
+
+class OpenAIConfig(BaseConfig):
+ """
+ Reference: https://platform.openai.com/docs/api-reference/chat/create
+
+ The class `OpenAIConfig` provides configuration for the OpenAI's Chat API interface. Below are the parameters:
+
+ - `frequency_penalty` (number or null): Defaults to 0. Allows a value between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, thereby minimizing repetition.
+
+ - `function_call` (string or object): This optional parameter controls how the model calls functions.
+
+ - `functions` (array): An optional parameter. It is a list of functions for which the model may generate JSON inputs.
+
+ - `logit_bias` (map): This optional parameter modifies the likelihood of specified tokens appearing in the completion.
+
+ - `max_tokens` (integer or null): This optional parameter helps to set the maximum number of tokens to generate in the chat completion. OpenAI has now deprecated in favor of max_completion_tokens, and is not compatible with o1 series models.
+
+ - `max_completion_tokens` (integer or null): An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens.
+
+ - `n` (integer or null): This optional parameter helps to set how many chat completion choices to generate for each input message.
+
+ - `presence_penalty` (number or null): Defaults to 0. It penalizes new tokens based on if they appear in the text so far, hence increasing the model's likelihood to talk about new topics.
+
+ - `stop` (string / array / null): Specifies up to 4 sequences where the API will stop generating further tokens.
+
+ - `temperature` (number or null): Defines the sampling temperature to use, varying between 0 and 2.
+
+ - `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling.
+ """
+
+ frequency_penalty: Optional[int] = None
+ function_call: Optional[Union[str, dict]] = None
+ functions: Optional[list] = None
+ logit_bias: Optional[dict] = None
+ max_completion_tokens: Optional[int] = None
+ max_tokens: Optional[int] = None
+ n: Optional[int] = None
+ presence_penalty: Optional[int] = None
+ stop: Optional[Union[str, list]] = None
+ temperature: Optional[int] = None
+ top_p: Optional[int] = None
+ response_format: Optional[dict] = None
+
+ def __init__(
+ self,
+ frequency_penalty: Optional[int] = None,
+ function_call: Optional[Union[str, dict]] = None,
+ functions: Optional[list] = None,
+ logit_bias: Optional[dict] = None,
+ max_completion_tokens: Optional[int] = None,
+ max_tokens: Optional[int] = None,
+ n: Optional[int] = None,
+ presence_penalty: Optional[int] = None,
+ stop: Optional[Union[str, list]] = None,
+ temperature: Optional[int] = None,
+ top_p: Optional[int] = None,
+ response_format: Optional[dict] = None,
+ ) -> None:
+ locals_ = locals().copy()
+ for key, value in locals_.items():
+ if key != "self" and value is not None:
+ setattr(self.__class__, key, value)
+
+ @classmethod
+ def get_config(cls):
+ return super().get_config()
+
+ def get_supported_openai_params(self, model: str) -> list:
+ """
+ This function returns the list
+ of supported openai parameters for a given OpenAI Model
+
+ - If O1 model, returns O1 supported params
+ - If gpt-audio model, returns gpt-audio supported params
+ - Else, returns gpt supported params
+
+ Args:
+ model (str): OpenAI model
+
+ Returns:
+ list: List of supported openai parameters
+ """
+ if openaiOSeriesConfig.is_model_o_series_model(model=model):
+ return openaiOSeriesConfig.get_supported_openai_params(model=model)
+ elif openAIGPT5Config.is_model_gpt_5_model(model=model):
+ return openAIGPT5Config.get_supported_openai_params(model=model)
+ elif litellm.openAIGPTAudioConfig.is_model_gpt_audio_model(model=model):
+ return litellm.openAIGPTAudioConfig.get_supported_openai_params(model=model)
+ else:
+ return litellm.openAIGPTConfig.get_supported_openai_params(model=model)
+
+ def _map_openai_params(
+ self, non_default_params: dict, optional_params: dict, model: str
+ ) -> dict:
+ supported_openai_params = self.get_supported_openai_params(model)
+ for param, value in non_default_params.items():
+ if param in supported_openai_params:
+ optional_params[param] = value
+ return optional_params
+
+ def _transform_messages(
+ self, messages: List[AllMessageValues], model: str
+ ) -> List[AllMessageValues]:
+ return messages
+
+ def map_openai_params(
+ self,
+ non_default_params: dict,
+ optional_params: dict,
+ model: str,
+ drop_params: bool,
+ ) -> dict:
+ """ """
+ if openaiOSeriesConfig.is_model_o_series_model(model=model):
+ return openaiOSeriesConfig.map_openai_params(
+ non_default_params=non_default_params,
+ optional_params=optional_params,
+ model=model,
+ drop_params=drop_params,
+ )
+ elif openAIGPT5Config.is_model_gpt_5_model(model=model):
+ return openAIGPT5Config.map_openai_params(
+ non_default_params=non_default_params,
+ optional_params=optional_params,
+ model=model,
+ drop_params=drop_params,
+ )
+ elif litellm.openAIGPTAudioConfig.is_model_gpt_audio_model(model=model):
+ return litellm.openAIGPTAudioConfig.map_openai_params(
+ non_default_params=non_default_params,
+ optional_params=optional_params,
+ model=model,
+ drop_params=drop_params,
+ )
+
+ return litellm.openAIGPTConfig.map_openai_params(
+ non_default_params=non_default_params,
+ optional_params=optional_params,
+ model=model,
+ drop_params=drop_params,
+ )
+
+ def get_error_class(
+ self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+ ) -> BaseLLMException:
+ return OpenAIError(
+ status_code=status_code,
+ message=error_message,
+ headers=headers,
+ )
+
+ def transform_request(
+ self,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ headers: dict,
+ ) -> dict:
+ messages = self._transform_messages(messages=messages, model=model)
+ return {"model": model, "messages": messages, **optional_params}
+
+ def transform_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ model_response: ModelResponse,
+ logging_obj: LiteLLMLoggingObj,
+ request_data: dict,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ encoding: Any,
+ api_key: Optional[str] = None,
+ json_mode: Optional[bool] = None,
+ ) -> ModelResponse:
+ logging_obj.post_call(original_response=raw_response.text)
+ logging_obj.model_call_details["response_headers"] = raw_response.headers
+ final_response_obj = cast(
+ ModelResponse,
+ convert_to_model_response_object(
+ response_object=raw_response.json(),
+ model_response_object=model_response,
+ hidden_params={"headers": raw_response.headers},
+ _response_headers=dict(raw_response.headers),
+ ),
+ )
+
+ return final_response_obj
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ messages: List[AllMessageValues],
+ optional_params: dict,
+ litellm_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ ) -> dict:
+ return {
+ "Authorization": f"Bearer {api_key}",
+ **headers,
+ }
+
+ def get_model_response_iterator(
+ self,
+ streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
+ sync_stream: bool,
+ json_mode: Optional[bool] = False,
+ ) -> Any:
+ return OpenAIChatCompletionResponseIterator(
+ streaming_response=streaming_response,
+ sync_stream=sync_stream,
+ json_mode=json_mode,
+ )
+
+
+class OpenAIChatCompletionResponseIterator(BaseModelResponseIterator):
+ def chunk_parser(self, chunk: dict) -> ModelResponseStream:
+ """
+ {'choices': [{'delta': {'content': '', 'role': 'assistant'}, 'finish_reason': None, 'index': 0, 'logprobs': None}], 'created': 1735763082, 'id': 'a83a2b0fbfaf4aab9c2c93cb8ba346d7', 'model': 'mistral-large', 'object': 'chat.completion.chunk'}
+ """
+ try:
+ return ModelResponseStream(**chunk)
+ except Exception as e:
+ raise e
+
+
+class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
+ def __init__(self) -> None:
+ super().__init__()
+
+ def _set_dynamic_params_on_client(
+ self,
+ client: Union[OpenAI, AsyncOpenAI],
+ organization: Optional[str] = None,
+ max_retries: Optional[int] = None,
+ ):
+ if organization is not None:
+ client.organization = organization
+ if max_retries is not None:
+ client.max_retries = max_retries
+
+ def _get_openai_client(
+ self,
+ is_async: bool,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ api_version: Optional[str] = None,
+ timeout: Union[float, httpx.Timeout] = httpx.Timeout(None),
+ max_retries: Optional[int] = DEFAULT_MAX_RETRIES,
+ organization: Optional[str] = None,
+ client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
+ shared_session: Optional["ClientSession"] = None,
+ ) -> Optional[Union[OpenAI, AsyncOpenAI]]:
+ client_initialization_params: Dict = locals()
+ if client is None:
+ if not isinstance(max_retries, int):
+ raise OpenAIError(
+ status_code=422,
+ message="max retries must be an int. Passed in value: {}".format(
+ max_retries
+ ),
+ )
+ cached_client = self.get_cached_openai_client(
+ client_initialization_params=client_initialization_params,
+ client_type="openai",
+ )
+
+ if cached_client:
+ if isinstance(cached_client, OpenAI) or isinstance(
+ cached_client, AsyncOpenAI
+ ):
+ return cached_client
+ if is_async:
+ _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI(
+ api_key=api_key,
+ base_url=api_base,
+ http_client=OpenAIChatCompletion._get_async_http_client(
+ shared_session=shared_session
+ ),
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ )
+ else:
+ _new_client = OpenAI(
+ api_key=api_key,
+ base_url=api_base,
+ http_client=OpenAIChatCompletion._get_sync_http_client(),
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ )
+
+ ## SAVE CACHE KEY
+ self.set_cached_openai_client(
+ openai_client=_new_client,
+ client_initialization_params=client_initialization_params,
+ client_type="openai",
+ )
+ return _new_client
+
+ else:
+ self._set_dynamic_params_on_client(
+ client=client,
+ organization=organization,
+ max_retries=max_retries,
+ )
+ return client
+
+ @track_llm_api_timing()
+ async def make_openai_chat_completion_request(
+ self,
+ openai_aclient: AsyncOpenAI,
+ data: dict,
+ timeout: Union[float, httpx.Timeout],
+ logging_obj: LiteLLMLoggingObj,
+ ) -> Tuple[dict, BaseModel]:
+ """
+ Helper to:
+ - call chat.completions.create.with_raw_response when litellm.return_response_headers is True
+ - call chat.completions.create by default
+ """
+ start_time = time.time()
+ try:
+ raw_response = (
+ await openai_aclient.chat.completions.with_raw_response.create(
+ **data, timeout=timeout
+ )
+ )
+ end_time = time.time()
+
+ if hasattr(raw_response, "headers"):
+ headers = dict(raw_response.headers)
+ else:
+ headers = {}
+ response = raw_response.parse()
+ if not data.get("stream") and not hasattr(response, "model_dump"):
+ raise OpenAIError(
+ status_code=500,
+ message=f"Empty or invalid response from LLM endpoint. Received: {response!r}. Check the reverse proxy or model server configuration.",
+ )
+ return headers, response
+ except openai.APITimeoutError as e:
+ end_time = time.time()
+ time_delta = round(end_time - start_time, 2)
+ e.message += f" - timeout value={timeout}, time taken={time_delta} seconds"
+ raise e
+ except Exception as e:
+ raise e
+
+ @track_llm_api_timing()
+ def make_sync_openai_chat_completion_request(
+ self,
+ openai_client: OpenAI,
+ data: dict,
+ timeout: Union[float, httpx.Timeout],
+ logging_obj: LiteLLMLoggingObj,
+ ) -> Tuple[dict, BaseModel]:
+ """
+ Helper to:
+ - call chat.completions.create.with_raw_response when litellm.return_response_headers is True
+ - call chat.completions.create by default
+ """
+ raw_response = None
+ try:
+ raw_response = openai_client.chat.completions.with_raw_response.create(
+ **data, timeout=timeout
+ )
+
+ if hasattr(raw_response, "headers"):
+ headers = dict(raw_response.headers)
+ else:
+ headers = {}
+ response = raw_response.parse()
+ if not data.get("stream") and not hasattr(response, "model_dump"):
+ raise OpenAIError(
+ status_code=500,
+ message=f"Empty or invalid response from LLM endpoint. Received: {response!r}. Check the reverse proxy or model server configuration.",
+ )
+ return headers, response
+ except OpenAIError:
+ raise
+ except Exception as e:
+ if raw_response is not None:
+ raise Exception(
+ "error - {}, Received response - {}, Type of response - {}".format(
+ e, raw_response, type(raw_response)
+ )
+ )
+ else:
+ raise e
+
+ async def _call_agentic_completion_hooks_openai(
+ self,
+ response: Any,
+ model: str,
+ messages: List[Dict],
+ optional_params: Dict,
+ logging_obj: LiteLLMLoggingObj,
+ stream: bool,
+ litellm_params: Dict,
+ ) -> Optional[Any]:
+ """
+ Call agentic completion hooks for all custom loggers (OpenAI Chat Completions API).
+
+ 1. Call async_should_run_chat_completion_agentic_loop to check if agentic loop is needed
+ 2. If yes, call async_run_chat_completion_agentic_loop to execute the loop
+
+ Returns the response from agentic loop, or None if no hook runs.
+ """
+ from litellm._logging import verbose_logger
+ from litellm.integrations.custom_logger import CustomLogger
+
+ callbacks = litellm.callbacks + (logging_obj.dynamic_success_callbacks or [])
+ # Avoid logging full callback objects to prevent leaking sensitive data
+ verbose_logger.debug("LiteLLM.AgenticHooks: callbacks_count=%s", len(callbacks))
+ tools = optional_params.get("tools", [])
+ # Avoid logging full tools payloads; they may contain sensitive parameters
+ verbose_logger.debug(
+ "LiteLLM.AgenticHooks: tools_count=%s",
+ len(tools) if isinstance(tools, list) else 1 if tools else 0,
+ )
+ # Get custom_llm_provider from litellm_params
+ custom_llm_provider = litellm_params.get("custom_llm_provider", "openai")
+
+ for callback in callbacks:
+ try:
+ if isinstance(callback, CustomLogger):
+ # Check if the callback has the chat completion agentic loop methods
+ if not hasattr(
+ callback, "async_should_run_chat_completion_agentic_loop"
+ ):
+ continue
+
+ # First: Check if agentic loop should run (using chat completion method)
+ (
+ should_run,
+ tool_calls,
+ ) = await callback.async_should_run_chat_completion_agentic_loop(
+ response=response,
+ model=model,
+ messages=messages,
+ tools=tools,
+ stream=stream,
+ custom_llm_provider=custom_llm_provider,
+ kwargs=litellm_params,
+ )
+
+ if should_run:
+ # Second: Execute agentic loop
+ kwargs_with_provider = (
+ litellm_params.copy() if litellm_params else {}
+ )
+ kwargs_with_provider[
+ "custom_llm_provider"
+ ] = custom_llm_provider
+
+ # For OpenAI Chat Completions, use the chat completion agentic loop method
+ agentic_response = (
+ await callback.async_run_chat_completion_agentic_loop(
+ tools=tool_calls,
+ model=model,
+ messages=messages,
+ response=response,
+ optional_params=optional_params,
+ logging_obj=logging_obj,
+ stream=stream,
+ kwargs=kwargs_with_provider,
+ )
+ )
+ # First hook that runs agentic loop wins
+ return agentic_response
+
+ except Exception as e:
+ verbose_logger.exception(
+ f"LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: {str(e)}"
+ )
+
+ return None
+
+ def mock_streaming(
+ self,
+ response: ModelResponse,
+ logging_obj: LiteLLMLoggingObj,
+ model: str,
+ stream_options: Optional[dict] = None,
+ ) -> CustomStreamWrapper:
+ completion_stream = MockResponseIterator(model_response=response)
+ streaming_response = CustomStreamWrapper(
+ completion_stream=completion_stream,
+ model=model,
+ custom_llm_provider="openai",
+ logging_obj=logging_obj,
+ stream_options=stream_options,
+ )
+
+ return streaming_response
+
+ def completion( # type: ignore # noqa: PLR0915
+ self,
+ model_response: ModelResponse,
+ timeout: Union[float, httpx.Timeout],
+ optional_params: dict,
+ litellm_params: dict,
+ logging_obj: Any,
+ model: Optional[str] = None,
+ messages: Optional[list] = None,
+ print_verbose: Optional[Callable] = None,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ api_version: Optional[str] = None,
+ dynamic_params: Optional[bool] = None,
+ azure_ad_token: Optional[str] = None,
+ acompletion: bool = False,
+ logger_fn=None,
+ headers: Optional[dict] = None,
+ custom_prompt_dict: dict = {},
+ client=None,
+ organization: Optional[str] = None,
+ custom_llm_provider: Optional[str] = None,
+ drop_params: Optional[bool] = None,
+ shared_session: Optional["ClientSession"] = None,
+ ):
+ super().completion(shared_session=shared_session)
+ try:
+ fake_stream: bool = False
+ inference_params = optional_params.copy()
+ stream_options: Optional[dict] = inference_params.pop(
+ "stream_options", None
+ )
+ stream: Optional[bool] = inference_params.pop("stream", False)
+ provider_config: Optional[BaseConfig] = None
+
+ if custom_llm_provider is not None and model is not None:
+ try:
+ provider_config = ProviderConfigManager.get_provider_chat_config(
+ model=model, provider=LlmProviders(custom_llm_provider)
+ )
+ except ValueError:
+ # JSON-configured providers may not be in LlmProviders enum
+ provider_config = None
+
+ if provider_config is None:
+ provider_config = OpenAIConfig()
+
+ if provider_config:
+ fake_stream = provider_config.should_fake_stream(
+ model=model, custom_llm_provider=custom_llm_provider, stream=stream
+ )
+
+ if headers:
+ inference_params["extra_headers"] = headers
+ if model is None or messages is None:
+ raise OpenAIError(status_code=422, message="Missing model or messages")
+
+ if not isinstance(timeout, float) and not isinstance(
+ timeout, httpx.Timeout
+ ):
+ raise OpenAIError(
+ status_code=422,
+ message="Timeout needs to be a float or httpx.Timeout",
+ )
+
+ if custom_llm_provider is not None and custom_llm_provider != "openai":
+ model_response.model = f"{custom_llm_provider}/{model}"
+
+ for _ in range(
+ 2
+ ): # if call fails due to alternating messages, retry with reformatted message
+ try:
+ max_retries = inference_params.pop("max_retries", 2)
+ if acompletion is True:
+ if stream is True and fake_stream is False:
+ return self.async_streaming(
+ logging_obj=logging_obj,
+ headers=headers,
+ messages=messages,
+ optional_params=inference_params,
+ litellm_params=litellm_params,
+ provider_config=provider_config,
+ model=model,
+ api_base=api_base,
+ api_key=api_key,
+ api_version=api_version,
+ timeout=timeout,
+ client=client,
+ max_retries=max_retries,
+ organization=organization,
+ drop_params=drop_params,
+ stream_options=stream_options,
+ shared_session=shared_session,
+ )
+ else:
+ return self.acompletion(
+ messages=messages,
+ optional_params=inference_params,
+ litellm_params=litellm_params,
+ provider_config=provider_config,
+ headers=headers,
+ model=model,
+ logging_obj=logging_obj,
+ model_response=model_response,
+ api_base=api_base,
+ api_key=api_key,
+ api_version=api_version,
+ timeout=timeout,
+ client=client,
+ max_retries=max_retries,
+ organization=organization,
+ drop_params=drop_params,
+ fake_stream=fake_stream,
+ shared_session=shared_session,
+ )
+
+ data = provider_config.transform_request(
+ model=model,
+ messages=messages,
+ optional_params=inference_params,
+ litellm_params=litellm_params,
+ headers=headers or {},
+ )
+ if stream is True and fake_stream is False:
+ return self.streaming(
+ logging_obj=logging_obj,
+ headers=headers,
+ data=data,
+ model=model,
+ api_base=api_base,
+ api_key=api_key,
+ api_version=api_version,
+ timeout=timeout,
+ client=client,
+ max_retries=max_retries,
+ organization=organization,
+ stream_options=stream_options,
+ )
+ else:
+ if not isinstance(max_retries, int):
+ raise OpenAIError(
+ status_code=422, message="max retries must be an int"
+ )
+ openai_client: OpenAI = self._get_openai_client( # type: ignore
+ is_async=False,
+ api_key=api_key,
+ api_base=api_base,
+ api_version=api_version,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ ## LOGGING
+ logging_obj.pre_call(
+ input=messages,
+ api_key=openai_client.api_key,
+ additional_args={
+ "headers": headers,
+ "api_base": openai_client._base_url._uri_reference,
+ "acompletion": acompletion,
+ "complete_input_dict": data,
+ },
+ )
+
+ (
+ headers,
+ response,
+ ) = self.make_sync_openai_chat_completion_request(
+ openai_client=openai_client,
+ data=data,
+ timeout=timeout,
+ logging_obj=logging_obj,
+ )
+
+ logging_obj.model_call_details["response_headers"] = headers
+ stringified_response = response.model_dump()
+ logging_obj.post_call(
+ input=messages,
+ api_key=api_key,
+ original_response=stringified_response,
+ additional_args={"complete_input_dict": data},
+ )
+
+ final_response_obj = convert_to_model_response_object(
+ response_object=stringified_response,
+ model_response_object=model_response,
+ _response_headers=headers,
+ )
+ if fake_stream is True:
+ return self.mock_streaming(
+ response=cast(ModelResponse, final_response_obj),
+ logging_obj=logging_obj,
+ model=model,
+ stream_options=stream_options,
+ )
+
+ return final_response_obj
+ except openai.UnprocessableEntityError as e:
+ ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
+ if litellm.drop_params is True or drop_params is True:
+ inference_params = drop_params_from_unprocessable_entity_error(
+ e, inference_params
+ )
+ else:
+ raise e
+ # e.message
+ except Exception as e:
+ if print_verbose is not None:
+ print_verbose(f"openai.py: Received openai error - {str(e)}")
+ if (
+ "Conversation roles must alternate user/assistant" in str(e)
+ or "user and assistant roles should be alternating" in str(e)
+ ) and messages is not None:
+ if print_verbose is not None:
+ print_verbose("openai.py: REFORMATS THE MESSAGE!")
+ # reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, add a blank 'user' or 'assistant' message to ensure compatibility
+ new_messages = []
+ for i in range(len(messages) - 1): # type: ignore
+ new_messages.append(messages[i])
+ if messages[i]["role"] == messages[i + 1]["role"]:
+ if messages[i]["role"] == "user":
+ new_messages.append(
+ {"role": "assistant", "content": ""}
+ )
+ else:
+ new_messages.append({"role": "user", "content": ""})
+ new_messages.append(messages[-1])
+ messages = new_messages
+ elif (
+ "Last message must have role `user`" in str(e)
+ ) and messages is not None:
+ new_messages = messages
+ new_messages.append({"role": "user", "content": ""})
+ messages = new_messages
+ elif "unknown field: parameter index is not a valid field" in str(
+ e
+ ):
+ litellm.remove_index_from_tool_calls(messages=messages)
+ else:
+ raise e
+ except OpenAIError as e:
+ raise e
+ except Exception as e:
+ status_code = getattr(e, "status_code", 500)
+ error_headers = getattr(e, "headers", None)
+ error_text = getattr(e, "text", str(e))
+ error_response = getattr(e, "response", None)
+ error_body = getattr(e, "body", None)
+ if error_headers is None and error_response:
+ error_headers = getattr(error_response, "headers", None)
+ raise OpenAIError(
+ status_code=status_code,
+ message=error_text,
+ headers=error_headers,
+ body=error_body,
+ )
+
+ async def acompletion(
+ self,
+ messages: list,
+ optional_params: dict,
+ litellm_params: dict,
+ provider_config: BaseConfig,
+ model: str,
+ model_response: ModelResponse,
+ logging_obj: LiteLLMLoggingObj,
+ timeout: Union[float, httpx.Timeout],
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ api_version: Optional[str] = None,
+ organization: Optional[str] = None,
+ client=None,
+ max_retries=None,
+ headers=None,
+ drop_params: Optional[bool] = None,
+ stream_options: Optional[dict] = None,
+ fake_stream: bool = False,
+ shared_session: Optional["ClientSession"] = None,
+ ):
+ response = None
+ data = await provider_config.async_transform_request(
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ headers=headers or {},
+ )
+ for _ in range(
+ 2
+ ): # if call fails due to alternating messages, retry with reformatted message
+ try:
+ openai_aclient: AsyncOpenAI = self._get_openai_client( # type: ignore
+ is_async=True,
+ api_key=api_key,
+ api_base=api_base,
+ api_version=api_version,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ shared_session=shared_session,
+ )
+
+ ## LOGGING
+ logging_obj.pre_call(
+ input=data["messages"],
+ api_key=openai_aclient.api_key,
+ additional_args={
+ "headers": {
+ "Authorization": f"Bearer {openai_aclient.api_key}"
+ },
+ "api_base": openai_aclient._base_url._uri_reference,
+ "acompletion": True,
+ "complete_input_dict": data,
+ },
+ )
+
+ headers, response = await self.make_openai_chat_completion_request(
+ openai_aclient=openai_aclient,
+ data=data,
+ timeout=timeout,
+ logging_obj=logging_obj,
+ )
+ stringified_response = response.model_dump()
+ logging_obj.post_call(
+ input=data["messages"],
+ api_key=api_key,
+ original_response=stringified_response,
+ additional_args={"complete_input_dict": data},
+ )
+ logging_obj.model_call_details["response_headers"] = headers
+ final_response_obj = convert_to_model_response_object(
+ response_object=stringified_response,
+ model_response_object=model_response,
+ hidden_params={"headers": headers},
+ _response_headers=headers,
+ )
+
+ # Call agentic completion hooks (e.g., for websearch_interception)
+ agentic_response = await self._call_agentic_completion_hooks_openai(
+ response=final_response_obj,
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ logging_obj=logging_obj,
+ stream=False,
+ litellm_params=litellm_params,
+ )
+
+ if agentic_response is not None:
+ final_response_obj = agentic_response
+
+ if fake_stream is True:
+ return self.mock_streaming(
+ response=cast(ModelResponse, final_response_obj),
+ logging_obj=logging_obj,
+ model=model,
+ stream_options=stream_options,
+ )
+
+ return final_response_obj
+ except openai.UnprocessableEntityError as e:
+ ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
+ if litellm.drop_params is True or drop_params is True:
+ data = drop_params_from_unprocessable_entity_error(e, data)
+ else:
+ raise e
+ # e.message
+ except Exception as e:
+ exception_response = getattr(e, "response", None)
+ status_code = getattr(e, "status_code", 500)
+ exception_body = getattr(e, "body", None)
+ error_headers = getattr(e, "headers", None)
+ if error_headers is None and exception_response:
+ error_headers = getattr(exception_response, "headers", None)
+ message = getattr(e, "message", str(e))
+
+ raise OpenAIError(
+ status_code=status_code,
+ message=message,
+ headers=error_headers,
+ body=exception_body,
+ )
+
+ def streaming(
+ self,
+ logging_obj,
+ timeout: Union[float, httpx.Timeout],
+ data: dict,
+ model: str,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ api_version: Optional[str] = None,
+ organization: Optional[str] = None,
+ client=None,
+ max_retries=None,
+ headers=None,
+ stream_options: Optional[dict] = None,
+ ):
+ data["stream"] = True
+ data.update(
+ self.get_stream_options(stream_options=stream_options, api_base=api_base)
+ )
+
+ openai_client: OpenAI = self._get_openai_client( # type: ignore
+ is_async=False,
+ api_key=api_key,
+ api_base=api_base,
+ api_version=api_version,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+ ## LOGGING
+ logging_obj.pre_call(
+ input=data["messages"],
+ api_key=api_key,
+ additional_args={
+ "headers": {"Authorization": f"Bearer {openai_client.api_key}"},
+ "api_base": openai_client._base_url._uri_reference,
+ "acompletion": False,
+ "complete_input_dict": data,
+ },
+ )
+ headers, response = self.make_sync_openai_chat_completion_request(
+ openai_client=openai_client,
+ data=data,
+ timeout=timeout,
+ logging_obj=logging_obj,
+ )
+
+ logging_obj.model_call_details["response_headers"] = headers
+ streamwrapper = CustomStreamWrapper(
+ completion_stream=response,
+ model=model,
+ custom_llm_provider="openai",
+ logging_obj=logging_obj,
+ stream_options=data.get("stream_options", None),
+ _response_headers=headers,
+ )
+ return streamwrapper
+
+ async def async_streaming(
+ self,
+ timeout: Union[float, httpx.Timeout],
+ messages: list,
+ optional_params: dict,
+ litellm_params: dict,
+ provider_config: BaseConfig,
+ model: str,
+ logging_obj: LiteLLMLoggingObj,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ api_version: Optional[str] = None,
+ organization: Optional[str] = None,
+ client=None,
+ max_retries=None,
+ headers=None,
+ drop_params: Optional[bool] = None,
+ stream_options: Optional[dict] = None,
+ shared_session: Optional["ClientSession"] = None,
+ ):
+ response = None
+ data = provider_config.transform_request(
+ model=model,
+ messages=messages,
+ optional_params=optional_params,
+ litellm_params=litellm_params,
+ headers=headers or {},
+ )
+ data["stream"] = True
+ data.update(
+ self.get_stream_options(stream_options=stream_options, api_base=api_base)
+ )
+ for _ in range(2):
+ try:
+ openai_aclient: AsyncOpenAI = self._get_openai_client( # type: ignore
+ is_async=True,
+ api_key=api_key,
+ api_base=api_base,
+ api_version=api_version,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ shared_session=shared_session,
+ )
+ ## LOGGING
+ logging_obj.pre_call(
+ input=data["messages"],
+ api_key=api_key,
+ additional_args={
+ "headers": headers,
+ "api_base": api_base,
+ "acompletion": True,
+ "complete_input_dict": data,
+ },
+ )
+
+ headers, response = await self.make_openai_chat_completion_request(
+ openai_aclient=openai_aclient,
+ data=data,
+ timeout=timeout,
+ logging_obj=logging_obj,
+ )
+ logging_obj.model_call_details["response_headers"] = headers
+ streamwrapper = CustomStreamWrapper(
+ completion_stream=response,
+ model=model,
+ custom_llm_provider="openai",
+ logging_obj=logging_obj,
+ stream_options=data.get("stream_options", None),
+ _response_headers=headers,
+ )
+ return streamwrapper
+ except openai.UnprocessableEntityError as e:
+ ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800
+ if litellm.drop_params is True or drop_params is True:
+ data = drop_params_from_unprocessable_entity_error(e, data)
+ else:
+ raise e
+ except (
+ Exception
+ ) as e: # need to exception handle here. async exceptions don't get caught in sync functions.
+ if isinstance(e, OpenAIError):
+ raise e
+
+ error_headers = getattr(e, "headers", None)
+ status_code = getattr(e, "status_code", 500)
+ error_response = getattr(e, "response", None)
+ exception_body = getattr(e, "body", None)
+ if error_headers is None and error_response:
+ error_headers = getattr(error_response, "headers", None)
+ if response is not None and hasattr(response, "text"):
+ raise OpenAIError(
+ status_code=status_code,
+ message=f"{str(e)}\n\nOriginal Response: {response.text}", # type: ignore
+ headers=error_headers,
+ body=exception_body,
+ )
+ else:
+ if type(e).__name__ == "ReadTimeout":
+ raise OpenAIError(
+ status_code=408,
+ message=f"{type(e).__name__}",
+ headers=error_headers,
+ body=exception_body,
+ )
+ elif hasattr(e, "status_code"):
+ raise OpenAIError(
+ status_code=getattr(e, "status_code", 500),
+ message=str(e),
+ headers=error_headers,
+ body=exception_body,
+ )
+ else:
+ raise OpenAIError(
+ status_code=500,
+ message=f"{str(e)}",
+ headers=error_headers,
+ body=exception_body,
+ )
+
+ def get_stream_options(
+ self, stream_options: Optional[dict], api_base: Optional[str]
+ ) -> dict:
+ """
+ Pass `stream_options` to the data dict for OpenAI requests
+ """
+ if stream_options is not None:
+ return {"stream_options": stream_options}
+ else:
+ # by default litellm will include usage for openai endpoints
+ if api_base is None or urlparse(api_base).hostname == "api.openai.com":
+ return {"stream_options": {"include_usage": True}}
+ return {}
+
+ # Embedding
+ @track_llm_api_timing()
+ async def make_openai_embedding_request(
+ self,
+ openai_aclient: AsyncOpenAI,
+ data: dict,
+ timeout: Union[float, httpx.Timeout],
+ logging_obj: LiteLLMLoggingObj,
+ ):
+ """
+ Helper to:
+ - call embeddings.create.with_raw_response when litellm.return_response_headers is True
+ - call embeddings.create by default
+ """
+ try:
+ raw_response = await openai_aclient.embeddings.with_raw_response.create(
+ **data, timeout=timeout
+ ) # type: ignore
+ headers = dict(raw_response.headers)
+ response = raw_response.parse()
+ return headers, response
+ except Exception as e:
+ raise e
+
+ @track_llm_api_timing()
+ def make_sync_openai_embedding_request(
+ self,
+ openai_client: OpenAI,
+ data: dict,
+ timeout: Union[float, httpx.Timeout],
+ logging_obj: LiteLLMLoggingObj,
+ ):
+ """
+ Helper to:
+ - call embeddings.create.with_raw_response when litellm.return_response_headers is True
+ - call embeddings.create by default
+ """
+ try:
+ raw_response = openai_client.embeddings.with_raw_response.create(
+ **data, timeout=timeout
+ ) # type: ignore
+
+ headers = dict(raw_response.headers)
+ response = raw_response.parse()
+ return headers, response
+ except Exception as e:
+ raise e
+
+ async def aembedding(
+ self,
+ input: list,
+ data: dict,
+ model_response: EmbeddingResponse,
+ timeout: float,
+ logging_obj: LiteLLMLoggingObj,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ client: Optional[AsyncOpenAI] = None,
+ max_retries=None,
+ shared_session: Optional["ClientSession"] = None,
+ ):
+ try:
+ openai_aclient: AsyncOpenAI = self._get_openai_client( # type: ignore
+ is_async=True,
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ client=client,
+ shared_session=shared_session,
+ )
+ headers, response = await self.make_openai_embedding_request(
+ openai_aclient=openai_aclient,
+ data=data,
+ timeout=timeout,
+ logging_obj=logging_obj,
+ )
+ logging_obj.model_call_details["response_headers"] = headers
+ stringified_response = response.model_dump()
+ ## LOGGING
+ logging_obj.post_call(
+ input=input,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data},
+ original_response=stringified_response,
+ )
+ returned_response: EmbeddingResponse = convert_to_model_response_object(
+ response_object=stringified_response,
+ model_response_object=model_response,
+ response_type="embedding",
+ _response_headers=headers,
+ ) # type: ignore
+ return returned_response
+ except OpenAIError as e:
+ ## LOGGING
+ logging_obj.post_call(
+ input=input,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data},
+ original_response=str(e),
+ )
+ raise e
+ except Exception as e:
+ ## LOGGING
+ logging_obj.post_call(
+ input=input,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data},
+ original_response=str(e),
+ )
+ status_code = getattr(e, "status_code", 500)
+ error_headers = getattr(e, "headers", None)
+ error_text = getattr(e, "text", str(e))
+ error_response = getattr(e, "response", None)
+ if error_headers is None and error_response:
+ error_headers = getattr(error_response, "headers", None)
+ raise OpenAIError(
+ status_code=status_code, message=error_text, headers=error_headers
+ )
+
+ def embedding( # type: ignore
+ self,
+ model: str,
+ input: list,
+ timeout: float,
+ logging_obj,
+ model_response: EmbeddingResponse,
+ optional_params: dict,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ client=None,
+ aembedding=None,
+ max_retries: Optional[int] = None,
+ shared_session: Optional["ClientSession"] = None,
+ ) -> EmbeddingResponse:
+ super().embedding()
+ try:
+ data = {"model": model, "input": input, **optional_params}
+ max_retries = max_retries or litellm.DEFAULT_MAX_RETRIES
+ if not isinstance(max_retries, int):
+ raise OpenAIError(status_code=422, message="max retries must be an int")
+ ## LOGGING
+ logging_obj.pre_call(
+ input=input,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data, "api_base": api_base},
+ )
+
+ if aembedding is True:
+ return self.aembedding( # type: ignore
+ data=data,
+ input=input,
+ logging_obj=logging_obj,
+ model_response=model_response,
+ api_base=api_base,
+ api_key=api_key,
+ timeout=timeout,
+ client=client,
+ max_retries=max_retries,
+ shared_session=shared_session,
+ )
+
+ openai_client: OpenAI = self._get_openai_client( # type: ignore
+ is_async=False,
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ client=client,
+ )
+
+ ## embedding CALL
+ headers: Optional[Dict] = None
+ headers, sync_embedding_response = self.make_sync_openai_embedding_request(
+ openai_client=openai_client,
+ data=data,
+ timeout=timeout,
+ logging_obj=logging_obj,
+ ) # type: ignore
+
+ ## LOGGING
+ logging_obj.model_call_details["response_headers"] = headers
+ logging_obj.post_call(
+ input=input,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data},
+ original_response=sync_embedding_response,
+ )
+ response: EmbeddingResponse = convert_to_model_response_object(
+ response_object=sync_embedding_response.model_dump(),
+ model_response_object=model_response,
+ _response_headers=headers,
+ response_type="embedding",
+ ) # type: ignore
+ return response
+ except OpenAIError as e:
+ raise e
+ except Exception as e:
+ status_code = getattr(e, "status_code", 500)
+ error_headers = getattr(e, "headers", None)
+ error_text = getattr(e, "text", str(e))
+ error_response = getattr(e, "response", None)
+ if error_headers is None and error_response:
+ error_headers = getattr(error_response, "headers", None)
+ raise OpenAIError(
+ status_code=status_code, message=error_text, headers=error_headers
+ )
+
+ async def aimage_generation(
+ self,
+ prompt: str,
+ data: dict,
+ model_response: ModelResponse,
+ timeout: float,
+ logging_obj: Any,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ client=None,
+ max_retries=None,
+ organization: Optional[str] = None,
+ headers: Optional[dict] = None,
+ ):
+ response = None
+ try:
+ openai_aclient = self._get_openai_client(
+ is_async=True,
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ if headers:
+ data["extra_headers"] = headers
+ response = await openai_aclient.images.generate(**data, timeout=timeout) # type: ignore
+ stringified_response = response.model_dump()
+ ## LOGGING
+ logging_obj.post_call(
+ input=prompt,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data},
+ original_response=stringified_response,
+ )
+ return convert_to_model_response_object(response_object=stringified_response, model_response_object=model_response, response_type="image_generation") # type: ignore
+ except Exception as e:
+ ## LOGGING
+ logging_obj.post_call(
+ input=prompt,
+ api_key=api_key,
+ original_response=str(e),
+ )
+ raise e
+
+ def image_generation(
+ self,
+ model: Optional[str],
+ prompt: str,
+ timeout: float,
+ optional_params: dict,
+ logging_obj: Any,
+ api_key: Optional[str] = None,
+ api_base: Optional[str] = None,
+ model_response: Optional[ImageResponse] = None,
+ client=None,
+ aimg_generation=None,
+ organization: Optional[str] = None,
+ headers: Optional[dict] = None,
+ ) -> ImageResponse:
+ data = {}
+ try:
+ data = {"model": model, "prompt": prompt, **optional_params}
+ max_retries = data.pop("max_retries", 2)
+ if not isinstance(max_retries, int):
+ raise OpenAIError(status_code=422, message="max retries must be an int")
+
+ if aimg_generation is True:
+ return self.aimage_generation(data=data, prompt=prompt, logging_obj=logging_obj, model_response=model_response, api_base=api_base, api_key=api_key, timeout=timeout, client=client, max_retries=max_retries, organization=organization, headers=headers) # type: ignore
+
+ openai_client: OpenAI = self._get_openai_client( # type: ignore
+ is_async=False,
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ ## LOGGING
+ logging_obj.pre_call(
+ input=prompt,
+ api_key=openai_client.api_key,
+ additional_args={
+ "headers": {"Authorization": f"Bearer {openai_client.api_key}"},
+ "api_base": openai_client._base_url._uri_reference,
+ "acompletion": True,
+ "complete_input_dict": data,
+ },
+ )
+
+ ## COMPLETION CALL
+ if headers:
+ data["extra_headers"] = headers
+ _response = openai_client.images.generate(**data, timeout=timeout) # type: ignore
+
+ response = _response.model_dump()
+ ## LOGGING
+ logging_obj.post_call(
+ input=prompt,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data},
+ original_response=response,
+ )
+ return convert_to_model_response_object(response_object=response, model_response_object=model_response, response_type="image_generation") # type: ignore
+ except OpenAIError as e:
+ ## LOGGING
+ logging_obj.post_call(
+ input=prompt,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data},
+ original_response=str(e),
+ )
+ raise e
+ except Exception as e:
+ ## LOGGING
+ logging_obj.post_call(
+ input=prompt,
+ api_key=api_key,
+ additional_args={"complete_input_dict": data},
+ original_response=str(e),
+ )
+ if hasattr(e, "status_code"):
+ raise OpenAIError(
+ status_code=getattr(e, "status_code", 500), message=str(e)
+ )
+ else:
+ raise OpenAIError(status_code=500, message=str(e))
+
+ def audio_speech(
+ self,
+ model: str,
+ input: str,
+ voice: str,
+ optional_params: dict,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ organization: Optional[str],
+ project: Optional[str],
+ max_retries: int,
+ timeout: Union[float, httpx.Timeout],
+ aspeech: Optional[bool] = None,
+ client=None,
+ shared_session: Optional["ClientSession"] = None,
+ ) -> HttpxBinaryResponseContent:
+ if aspeech is not None and aspeech is True:
+ return self.async_audio_speech(
+ model=model,
+ input=input,
+ voice=voice,
+ optional_params=optional_params,
+ api_key=api_key,
+ api_base=api_base,
+ organization=organization,
+ project=project,
+ max_retries=max_retries,
+ timeout=timeout,
+ client=client,
+ shared_session=shared_session,
+ ) # type: ignore
+
+ openai_client = self._get_openai_client(
+ is_async=False,
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ client=client,
+ shared_session=shared_session,
+ )
+
+ response = cast(OpenAI, openai_client).audio.speech.create(
+ model=model,
+ voice=voice, # type: ignore
+ input=input,
+ **optional_params,
+ )
+ return HttpxBinaryResponseContent(response=response.response)
+
+ async def async_audio_speech(
+ self,
+ model: str,
+ input: str,
+ voice: str,
+ optional_params: dict,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ organization: Optional[str],
+ project: Optional[str],
+ max_retries: int,
+ timeout: Union[float, httpx.Timeout],
+ client=None,
+ shared_session: Optional["ClientSession"] = None,
+ ) -> HttpxBinaryResponseContent:
+ openai_client = cast(
+ AsyncOpenAI,
+ self._get_openai_client(
+ is_async=True,
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ client=client,
+ shared_session=shared_session,
+ ),
+ )
+
+ response = await openai_client.audio.speech.create(
+ model=model,
+ voice=voice, # type: ignore
+ input=input,
+ **optional_params,
+ )
+
+ return HttpxBinaryResponseContent(response=response.response)
+
+
+class OpenAIFilesAPI(BaseLLM):
+ """
+ OpenAI methods to support for batches
+ - create_file()
+ - retrieve_file()
+ - list_files()
+ - delete_file()
+ - file_content()
+ - update_file()
+ """
+
+ def __init__(self) -> None:
+ super().__init__()
+
+ def get_openai_client(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
+ _is_async: bool = False,
+ ) -> Optional[Union[OpenAI, AsyncOpenAI]]:
+ received_args = locals()
+ openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = None
+ if client is None:
+ data = {}
+ for k, v in received_args.items():
+ if k == "self" or k == "client" or k == "_is_async":
+ pass
+ elif k == "api_base" and v is not None:
+ data["base_url"] = v
+ elif v is not None:
+ data[k] = v
+ if _is_async is True:
+ openai_client = AsyncOpenAI(**data)
+ else:
+ openai_client = OpenAI(**data) # type: ignore
+ else:
+ openai_client = client
+
+ return openai_client
+
+ async def acreate_file(
+ self,
+ create_file_data: CreateFileRequest,
+ openai_client: AsyncOpenAI,
+ ) -> OpenAIFileObject:
+ response = await openai_client.files.create(**create_file_data) # type: ignore[arg-type]
+ return OpenAIFileObject(**response.model_dump())
+
+ def create_file(
+ self,
+ _is_async: bool,
+ create_file_data: CreateFileRequest,
+ api_base: str,
+ api_key: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
+ ) -> Union[OpenAIFileObject, Coroutine[Any, Any, OpenAIFileObject]]:
+ openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, AsyncOpenAI):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.acreate_file( # type: ignore
+ create_file_data=create_file_data, openai_client=openai_client
+ )
+ response = cast(OpenAI, openai_client).files.create(**create_file_data) # type: ignore[arg-type]
+ return OpenAIFileObject(**response.model_dump())
+
+ async def afile_content(
+ self,
+ file_content_request: FileContentRequest,
+ openai_client: AsyncOpenAI,
+ ) -> HttpxBinaryResponseContent:
+ response = await openai_client.files.content(**file_content_request)
+ return HttpxBinaryResponseContent(response=response.response)
+
+ def file_content(
+ self,
+ _is_async: bool,
+ file_content_request: FileContentRequest,
+ api_base: str,
+ api_key: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
+ ) -> Union[
+ HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]
+ ]:
+ openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, AsyncOpenAI):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.afile_content( # type: ignore
+ file_content_request=file_content_request,
+ openai_client=openai_client,
+ )
+ response = cast(OpenAI, openai_client).files.content(**file_content_request)
+
+ return HttpxBinaryResponseContent(response=response.response)
+
+ async def aretrieve_file(
+ self,
+ file_id: str,
+ openai_client: AsyncOpenAI,
+ ) -> FileObject:
+ response = await openai_client.files.retrieve(file_id=file_id)
+ return response
+
+ def retrieve_file(
+ self,
+ _is_async: bool,
+ file_id: str,
+ api_base: str,
+ api_key: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
+ ):
+ openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, AsyncOpenAI):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.aretrieve_file( # type: ignore
+ file_id=file_id,
+ openai_client=openai_client,
+ )
+ response = openai_client.files.retrieve(file_id=file_id)
+
+ return response
+
+ async def adelete_file(
+ self,
+ file_id: str,
+ openai_client: AsyncOpenAI,
+ ) -> FileDeleted:
+ response = await openai_client.files.delete(file_id=file_id)
+ return response
+
+ def delete_file(
+ self,
+ _is_async: bool,
+ file_id: str,
+ api_base: str,
+ api_key: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
+ ):
+ openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, AsyncOpenAI):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.adelete_file( # type: ignore
+ file_id=file_id,
+ openai_client=openai_client,
+ )
+ response = openai_client.files.delete(file_id=file_id)
+
+ return response
+
+ async def alist_files(
+ self,
+ openai_client: AsyncOpenAI,
+ purpose: Optional[str] = None,
+ ):
+ if isinstance(purpose, str):
+ response = await openai_client.files.list(purpose=purpose)
+ else:
+ response = await openai_client.files.list()
+ return response
+
+ def list_files(
+ self,
+ _is_async: bool,
+ api_base: str,
+ api_key: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ purpose: Optional[str] = None,
+ client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
+ ):
+ openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, AsyncOpenAI):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.alist_files( # type: ignore
+ purpose=purpose,
+ openai_client=openai_client,
+ )
+
+ if isinstance(purpose, str):
+ response = openai_client.files.list(purpose=purpose)
+ else:
+ response = openai_client.files.list()
+
+ return response
+
+
+class OpenAIBatchesAPI(BaseLLM):
+ """
+ OpenAI methods to support for batches
+ - create_batch()
+ - retrieve_batch()
+ - cancel_batch()
+ - list_batch()
+ """
+
+ def __init__(self) -> None:
+ super().__init__()
+
+ def get_openai_client(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
+ _is_async: bool = False,
+ ) -> Optional[Union[OpenAI, AsyncOpenAI]]:
+ received_args = locals()
+ openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = None
+ if client is None:
+ data = {}
+ for k, v in received_args.items():
+ if k == "self" or k == "client" or k == "_is_async":
+ pass
+ elif k == "api_base" and v is not None:
+ data["base_url"] = v
+ elif v is not None:
+ data[k] = v
+ if _is_async is True:
+ openai_client = AsyncOpenAI(**data)
+ else:
+ openai_client = OpenAI(**data) # type: ignore
+ else:
+ openai_client = client
+
+ return openai_client
+
+ async def acreate_batch(
+ self,
+ create_batch_data: CreateBatchRequest,
+ openai_client: AsyncOpenAI,
+ ) -> LiteLLMBatch:
+ response = await openai_client.batches.create(**create_batch_data) # type: ignore[arg-type]
+ return LiteLLMBatch(**response.model_dump())
+
+ def create_batch(
+ self,
+ _is_async: bool,
+ create_batch_data: CreateBatchRequest,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
+ ) -> Union[LiteLLMBatch, Coroutine[Any, Any, LiteLLMBatch]]:
+ openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, AsyncOpenAI):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.acreate_batch( # type: ignore
+ create_batch_data=create_batch_data, openai_client=openai_client
+ )
+ response = cast(OpenAI, openai_client).batches.create(**create_batch_data) # type: ignore[arg-type]
+
+ return LiteLLMBatch(**response.model_dump())
+
+ async def aretrieve_batch(
+ self,
+ retrieve_batch_data: RetrieveBatchRequest,
+ openai_client: AsyncOpenAI,
+ ) -> LiteLLMBatch:
+ verbose_logger.debug("retrieving batch, args= %s", retrieve_batch_data)
+ response = await openai_client.batches.retrieve(**retrieve_batch_data) # type: ignore[arg-type]
+ return LiteLLMBatch(**response.model_dump())
+
+ def retrieve_batch(
+ self,
+ _is_async: bool,
+ retrieve_batch_data: RetrieveBatchRequest,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[OpenAI] = None,
+ ):
+ openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, AsyncOpenAI):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.aretrieve_batch( # type: ignore
+ retrieve_batch_data=retrieve_batch_data, openai_client=openai_client
+ )
+ response = cast(OpenAI, openai_client).batches.retrieve(**retrieve_batch_data) # type: ignore[arg-type]
+ return LiteLLMBatch(**response.model_dump())
+
+ async def acancel_batch(
+ self,
+ cancel_batch_data: CancelBatchRequest,
+ openai_client: AsyncOpenAI,
+ ) -> LiteLLMBatch:
+ verbose_logger.debug("async cancelling batch, args= %s", cancel_batch_data)
+ response = await openai_client.batches.cancel(**cancel_batch_data)
+ return LiteLLMBatch(**response.model_dump())
+
+ def cancel_batch(
+ self,
+ _is_async: bool,
+ cancel_batch_data: CancelBatchRequest,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[OpenAI] = None,
+ ):
+ openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, AsyncOpenAI):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.acancel_batch( # type: ignore
+ cancel_batch_data=cancel_batch_data, openai_client=openai_client
+ )
+
+ # At this point, openai_client is guaranteed to be a sync OpenAI client
+ if not isinstance(openai_client, OpenAI):
+ raise ValueError(
+ "OpenAI client is not an instance of OpenAI. Make sure you passed a sync OpenAI client."
+ )
+ response = openai_client.batches.cancel(**cancel_batch_data)
+ return LiteLLMBatch(**response.model_dump())
+
+ async def alist_batches(
+ self,
+ openai_client: AsyncOpenAI,
+ after: Optional[str] = None,
+ limit: Optional[int] = None,
+ ):
+ verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit)
+ response = await openai_client.batches.list(after=after, limit=limit) # type: ignore
+ return response
+
+ def list_batches(
+ self,
+ _is_async: bool,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ after: Optional[str] = None,
+ limit: Optional[int] = None,
+ client: Optional[OpenAI] = None,
+ ):
+ openai_client: Optional[Union[OpenAI, AsyncOpenAI]] = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ _is_async=_is_async,
+ )
+ if openai_client is None:
+ raise ValueError(
+ "OpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment."
+ )
+
+ if _is_async is True:
+ if not isinstance(openai_client, AsyncOpenAI):
+ raise ValueError(
+ "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
+ )
+ return self.alist_batches( # type: ignore
+ openai_client=openai_client, after=after, limit=limit
+ )
+ response = openai_client.batches.list(after=after, limit=limit) # type: ignore
+ return response
+
+
+class OpenAIAssistantsAPI(BaseLLM):
+ def __init__(self) -> None:
+ super().__init__()
+
+ def get_openai_client(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[OpenAI] = None,
+ ) -> OpenAI:
+ received_args = locals()
+ if client is None:
+ data = {}
+ for k, v in received_args.items():
+ if k == "self" or k == "client":
+ pass
+ elif k == "api_base" and v is not None:
+ data["base_url"] = v
+ elif v is not None:
+ data[k] = v
+ openai_client = OpenAI(**data) # type: ignore
+ else:
+ openai_client = client
+
+ return openai_client
+
+ def async_get_openai_client(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI] = None,
+ ) -> AsyncOpenAI:
+ received_args = locals()
+ if client is None:
+ data = {}
+ for k, v in received_args.items():
+ if k == "self" or k == "client":
+ pass
+ elif k == "api_base" and v is not None:
+ data["base_url"] = v
+ elif v is not None:
+ data[k] = v
+ openai_client = AsyncOpenAI(**data) # type: ignore
+ else:
+ openai_client = client
+
+ return openai_client
+
+ ### ASSISTANTS ###
+
+ async def async_get_assistants(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI],
+ order: Optional[str] = "desc",
+ limit: Optional[int] = 20,
+ before: Optional[str] = None,
+ after: Optional[str] = None,
+ ) -> AsyncCursorPage[Assistant]:
+ openai_client = self.async_get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+ request_params = {
+ "order": order,
+ "limit": limit,
+ }
+ if before:
+ request_params["before"] = before
+ if after:
+ request_params["after"] = after
+
+ response = await openai_client.beta.assistants.list(**request_params) # type: ignore
+
+ return response
+
+ # fmt: off
+
+ @overload
+ def get_assistants(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI],
+ aget_assistants: Literal[True],
+ ) -> Coroutine[None, None, AsyncCursorPage[Assistant]]:
+ ...
+
+ @overload
+ def get_assistants(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[OpenAI],
+ aget_assistants: Optional[Literal[False]],
+ ) -> SyncCursorPage[Assistant]:
+ ...
+
+ # fmt: on
+
+ def get_assistants(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client=None,
+ aget_assistants=None,
+ order: Optional[str] = "desc",
+ limit: Optional[int] = 20,
+ before: Optional[str] = None,
+ after: Optional[str] = None,
+ ):
+ if aget_assistants is not None and aget_assistants is True:
+ return self.async_get_assistants(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+ openai_client = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ request_params = {
+ "order": order,
+ "limit": limit,
+ }
+
+ if before:
+ request_params["before"] = before
+ if after:
+ request_params["after"] = after
+
+ response = openai_client.beta.assistants.list(**request_params) # type: ignore
+
+ return response
+
+ # Create Assistant
+ async def async_create_assistants(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI],
+ create_assistant_data: dict,
+ ) -> Assistant:
+ openai_client = self.async_get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ response = await openai_client.beta.assistants.create(**create_assistant_data)
+
+ return response
+
+ def create_assistants(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ create_assistant_data: dict,
+ client=None,
+ async_create_assistants=None,
+ ):
+ if async_create_assistants is not None and async_create_assistants is True:
+ return self.async_create_assistants(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ create_assistant_data=create_assistant_data,
+ )
+ openai_client = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ response = openai_client.beta.assistants.create(**create_assistant_data)
+ return response
+
+ # Delete Assistant
+ async def async_delete_assistant(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI],
+ assistant_id: str,
+ ) -> AssistantDeleted:
+ openai_client = self.async_get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ response = await openai_client.beta.assistants.delete(assistant_id=assistant_id)
+
+ return response
+
+ def delete_assistant(
+ self,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ assistant_id: str,
+ client=None,
+ async_delete_assistants=None,
+ ):
+ if async_delete_assistants is not None and async_delete_assistants is True:
+ return self.async_delete_assistant(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ assistant_id=assistant_id,
+ )
+ openai_client = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ response = openai_client.beta.assistants.delete(assistant_id=assistant_id)
+ return response
+
+ ### MESSAGES ###
+
+ async def a_add_message(
+ self,
+ thread_id: str,
+ message_data: dict,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI] = None,
+ ) -> OpenAIMessage:
+ openai_client = self.async_get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ thread_message: OpenAIMessage = await openai_client.beta.threads.messages.create( # type: ignore
+ thread_id, **message_data # type: ignore
+ )
+
+ response_obj: Optional[OpenAIMessage] = None
+ if getattr(thread_message, "status", None) is None:
+ thread_message.status = "completed"
+ response_obj = OpenAIMessage(**thread_message.dict())
+ else:
+ response_obj = OpenAIMessage(**thread_message.dict())
+ return response_obj
+
+ # fmt: off
+
+ @overload
+ def add_message(
+ self,
+ thread_id: str,
+ message_data: dict,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI],
+ a_add_message: Literal[True],
+ ) -> Coroutine[None, None, OpenAIMessage]:
+ ...
+
+ @overload
+ def add_message(
+ self,
+ thread_id: str,
+ message_data: dict,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[OpenAI],
+ a_add_message: Optional[Literal[False]],
+ ) -> OpenAIMessage:
+ ...
+
+ # fmt: on
+
+ def add_message(
+ self,
+ thread_id: str,
+ message_data: dict,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client=None,
+ a_add_message: Optional[bool] = None,
+ ):
+ if a_add_message is not None and a_add_message is True:
+ return self.a_add_message(
+ thread_id=thread_id,
+ message_data=message_data,
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+ openai_client = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ thread_message: OpenAIMessage = openai_client.beta.threads.messages.create( # type: ignore
+ thread_id, **message_data # type: ignore
+ )
+
+ response_obj: Optional[OpenAIMessage] = None
+ if getattr(thread_message, "status", None) is None:
+ thread_message.status = "completed"
+ response_obj = OpenAIMessage(**thread_message.dict())
+ else:
+ response_obj = OpenAIMessage(**thread_message.dict())
+ return response_obj
+
+ async def async_get_messages(
+ self,
+ thread_id: str,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI] = None,
+ ) -> AsyncCursorPage[OpenAIMessage]:
+ openai_client = self.async_get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ response = await openai_client.beta.threads.messages.list(thread_id=thread_id)
+
+ return response
+
+ # fmt: off
+
+ @overload
+ def get_messages(
+ self,
+ thread_id: str,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI],
+ aget_messages: Literal[True],
+ ) -> Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
+ ...
+
+ @overload
+ def get_messages(
+ self,
+ thread_id: str,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[OpenAI],
+ aget_messages: Optional[Literal[False]],
+ ) -> SyncCursorPage[OpenAIMessage]:
+ ...
+
+ # fmt: on
+
+ def get_messages(
+ self,
+ thread_id: str,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client=None,
+ aget_messages=None,
+ ):
+ if aget_messages is not None and aget_messages is True:
+ return self.async_get_messages(
+ thread_id=thread_id,
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+ openai_client = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ response = openai_client.beta.threads.messages.list(thread_id=thread_id)
+
+ return response
+
+ ### THREADS ###
+
+ async def async_create_thread(
+ self,
+ metadata: Optional[dict],
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI],
+ messages: Optional[Iterable[OpenAICreateThreadParamsMessage]],
+ ) -> Thread:
+ openai_client = self.async_get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ data = {}
+ if messages is not None:
+ data["messages"] = messages # type: ignore
+ if metadata is not None:
+ data["metadata"] = metadata # type: ignore
+
+ message_thread = await openai_client.beta.threads.create(**data) # type: ignore
+
+ return Thread(**message_thread.dict())
+
+ # fmt: off
+
+ @overload
+ def create_thread(
+ self,
+ metadata: Optional[dict],
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ messages: Optional[Iterable[OpenAICreateThreadParamsMessage]],
+ client: Optional[AsyncOpenAI],
+ acreate_thread: Literal[True],
+ ) -> Coroutine[None, None, Thread]:
+ ...
+
+ @overload
+ def create_thread(
+ self,
+ metadata: Optional[dict],
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ messages: Optional[Iterable[OpenAICreateThreadParamsMessage]],
+ client: Optional[OpenAI],
+ acreate_thread: Optional[Literal[False]],
+ ) -> Thread:
+ ...
+
+ # fmt: on
+
+ def create_thread(
+ self,
+ metadata: Optional[dict],
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ messages: Optional[Iterable[OpenAICreateThreadParamsMessage]],
+ client=None,
+ acreate_thread=None,
+ ):
+ """
+ Here's an example:
+ ```
+ from litellm.llms.openai.openai import OpenAIAssistantsAPI, MessageData
+
+ # create thread
+ message: MessageData = {"role": "user", "content": "Hey, how's it going?"}
+ openai_api.create_thread(messages=[message])
+ ```
+ """
+ if acreate_thread is not None and acreate_thread is True:
+ return self.async_create_thread(
+ metadata=metadata,
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ messages=messages,
+ )
+ openai_client = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ data = {}
+ if messages is not None:
+ data["messages"] = messages # type: ignore
+ if metadata is not None:
+ data["metadata"] = metadata # type: ignore
+
+ message_thread = openai_client.beta.threads.create(**data) # type: ignore
+
+ return Thread(**message_thread.dict())
+
+ async def async_get_thread(
+ self,
+ thread_id: str,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI],
+ ) -> Thread:
+ openai_client = self.async_get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ response = await openai_client.beta.threads.retrieve(thread_id=thread_id)
+
+ return Thread(**response.dict())
+
+ # fmt: off
+
+ @overload
+ def get_thread(
+ self,
+ thread_id: str,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI],
+ aget_thread: Literal[True],
+ ) -> Coroutine[None, None, Thread]:
+ ...
+
+ @overload
+ def get_thread(
+ self,
+ thread_id: str,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[OpenAI],
+ aget_thread: Optional[Literal[False]],
+ ) -> Thread:
+ ...
+
+ # fmt: on
+
+ def get_thread(
+ self,
+ thread_id: str,
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client=None,
+ aget_thread=None,
+ ):
+ if aget_thread is not None and aget_thread is True:
+ return self.async_get_thread(
+ thread_id=thread_id,
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+ openai_client = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ response = openai_client.beta.threads.retrieve(thread_id=thread_id)
+
+ return Thread(**response.dict())
+
+ def delete_thread(self):
+ pass
+
+ ### RUNS ###
+
+ async def arun_thread(
+ self,
+ thread_id: str,
+ assistant_id: str,
+ additional_instructions: Optional[str],
+ instructions: Optional[str],
+ metadata: Optional[Dict],
+ model: Optional[str],
+ stream: Optional[bool],
+ tools: Optional[Iterable[AssistantToolParam]],
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client: Optional[AsyncOpenAI],
+ ) -> Run:
+ openai_client = self.async_get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ response = await openai_client.beta.threads.runs.create_and_poll( # type: ignore
+ thread_id=thread_id,
+ assistant_id=assistant_id,
+ additional_instructions=additional_instructions,
+ instructions=instructions,
+ metadata=metadata,
+ model=model,
+ tools=tools,
+ )
+
+ return response
+
+ def async_run_thread_stream(
+ self,
+ client: AsyncOpenAI,
+ thread_id: str,
+ assistant_id: str,
+ additional_instructions: Optional[str],
+ instructions: Optional[str],
+ metadata: Optional[Dict],
+ model: Optional[str],
+ tools: Optional[Iterable[AssistantToolParam]],
+ event_handler: Optional[AssistantEventHandler],
+ ) -> AsyncAssistantStreamManager[AsyncAssistantEventHandler]:
+ data: Dict[str, Any] = {
+ "thread_id": thread_id,
+ "assistant_id": assistant_id,
+ "additional_instructions": additional_instructions,
+ "instructions": instructions,
+ "metadata": metadata,
+ "model": model,
+ "tools": tools,
+ }
+ if event_handler is not None:
+ data["event_handler"] = event_handler
+ return client.beta.threads.runs.stream(**data) # type: ignore
+
+ def run_thread_stream(
+ self,
+ client: OpenAI,
+ thread_id: str,
+ assistant_id: str,
+ additional_instructions: Optional[str],
+ instructions: Optional[str],
+ metadata: Optional[Dict],
+ model: Optional[str],
+ tools: Optional[Iterable[AssistantToolParam]],
+ event_handler: Optional[AssistantEventHandler],
+ ) -> AssistantStreamManager[AssistantEventHandler]:
+ data: Dict[str, Any] = {
+ "thread_id": thread_id,
+ "assistant_id": assistant_id,
+ "additional_instructions": additional_instructions,
+ "instructions": instructions,
+ "metadata": metadata,
+ "model": model,
+ "tools": tools,
+ }
+ if event_handler is not None:
+ data["event_handler"] = event_handler
+ return client.beta.threads.runs.stream(**data) # type: ignore
+
+ # fmt: off
+
+ @overload
+ def run_thread(
+ self,
+ thread_id: str,
+ assistant_id: str,
+ additional_instructions: Optional[str],
+ instructions: Optional[str],
+ metadata: Optional[Dict],
+ model: Optional[str],
+ stream: Optional[bool],
+ tools: Optional[Iterable[AssistantToolParam]],
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client,
+ arun_thread: Literal[True],
+ event_handler: Optional[AssistantEventHandler],
+ ) -> Coroutine[None, None, Run]:
+ ...
+
+ @overload
+ def run_thread(
+ self,
+ thread_id: str,
+ assistant_id: str,
+ additional_instructions: Optional[str],
+ instructions: Optional[str],
+ metadata: Optional[Dict],
+ model: Optional[str],
+ stream: Optional[bool],
+ tools: Optional[Iterable[AssistantToolParam]],
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client,
+ arun_thread: Optional[Literal[False]],
+ event_handler: Optional[AssistantEventHandler],
+ ) -> Run:
+ ...
+
+ # fmt: on
+
+ def run_thread(
+ self,
+ thread_id: str,
+ assistant_id: str,
+ additional_instructions: Optional[str],
+ instructions: Optional[str],
+ metadata: Optional[Dict],
+ model: Optional[str],
+ stream: Optional[bool],
+ tools: Optional[Iterable[AssistantToolParam]],
+ api_key: Optional[str],
+ api_base: Optional[str],
+ timeout: Union[float, httpx.Timeout],
+ max_retries: Optional[int],
+ organization: Optional[str],
+ client=None,
+ arun_thread=None,
+ event_handler: Optional[AssistantEventHandler] = None,
+ ):
+ if arun_thread is not None and arun_thread is True:
+ if stream is not None and stream is True:
+ _client = self.async_get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+ return self.async_run_thread_stream(
+ client=_client,
+ thread_id=thread_id,
+ assistant_id=assistant_id,
+ additional_instructions=additional_instructions,
+ instructions=instructions,
+ metadata=metadata,
+ model=model,
+ tools=tools,
+ event_handler=event_handler,
+ )
+ return self.arun_thread(
+ thread_id=thread_id,
+ assistant_id=assistant_id,
+ additional_instructions=additional_instructions,
+ instructions=instructions,
+ metadata=metadata,
+ model=model,
+ stream=stream,
+ tools=tools,
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+ openai_client = self.get_openai_client(
+ api_key=api_key,
+ api_base=api_base,
+ timeout=timeout,
+ max_retries=max_retries,
+ organization=organization,
+ client=client,
+ )
+
+ if stream is not None and stream is True:
+ return self.run_thread_stream(
+ client=openai_client,
+ thread_id=thread_id,
+ assistant_id=assistant_id,
+ additional_instructions=additional_instructions,
+ instructions=instructions,
+ metadata=metadata,
+ model=model,
+ tools=tools,
+ event_handler=event_handler,
+ )
+
+ response = openai_client.beta.threads.runs.create_and_poll( # type: ignore
+ thread_id=thread_id,
+ assistant_id=assistant_id,
+ additional_instructions=additional_instructions,
+ instructions=instructions,
+ metadata=metadata,
+ model=model,
+ tools=tools,
+ )
+
+ return response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/realtime/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/realtime/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..c04857fc25f883ad97f9ce3b257c18f4c20c97e5
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/realtime/handler.py
@@ -0,0 +1,167 @@
+"""
+This file contains the calling OpenAI's `/v1/realtime` endpoint.
+
+This requires websockets, and is currently only supported on LiteLLM Proxy.
+"""
+
+from typing import Any, Optional, cast
+
+from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
+from litellm.types.realtime import RealtimeQueryParams
+
+from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
+from ....litellm_core_utils.realtime_streaming import RealTimeStreaming
+from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
+from ..openai import OpenAIChatCompletion
+
+
+class OpenAIRealtime(OpenAIChatCompletion):
+ """
+ Base handler for OpenAI-compatible realtime WebSocket connections.
+
+ Subclasses can override template methods to customize:
+ - _get_default_api_base(): Default API base URL
+ - _get_additional_headers(): Extra headers beyond Authorization
+ - _get_ssl_config(): SSL configuration for WebSocket connection
+ """
+
+ def _get_default_api_base(self) -> str:
+ """
+ Get the default API base URL for this provider.
+ Override this in subclasses to set provider-specific defaults.
+ """
+ return "https://api.openai.com/"
+
+ def _get_additional_headers(self, api_key: str) -> dict:
+ """
+ Get additional headers beyond Authorization.
+ Override this in subclasses to customize headers (e.g., remove OpenAI-Beta).
+
+ Args:
+ api_key: API key for authentication
+
+ Returns:
+ Dictionary of additional headers
+ """
+ return {
+ "Authorization": f"Bearer {api_key}",
+ "OpenAI-Beta": "realtime=v1",
+ }
+
+ def _get_ssl_config(self, url: str) -> Any:
+ """
+ Get SSL configuration for WebSocket connection.
+ Override this in subclasses to customize SSL behavior.
+
+ Args:
+ url: WebSocket URL (ws:// or wss://)
+
+ Returns:
+ SSL configuration (None, True, or SSLContext)
+ """
+ if url.startswith("ws://"):
+ return None
+
+ # Use the shared SSL context which respects custom CA certs and SSL settings
+ ssl_config = get_shared_realtime_ssl_context()
+
+ # If ssl_config is False (ssl_verify=False), websockets library needs True instead
+ # to establish connection without verification (False would fail)
+ if ssl_config is False:
+ return True
+
+ return ssl_config
+
+ def _construct_url(self, api_base: str, query_params: RealtimeQueryParams) -> str:
+ """
+ Construct the backend websocket URL with all query parameters (including 'model').
+ """
+ from httpx import URL
+
+ api_base = api_base.replace("https://", "wss://")
+ api_base = api_base.replace("http://", "ws://")
+ url = URL(api_base)
+ # Set the correct path
+ url = url.copy_with(path="/v1/realtime")
+ # Include all query parameters including 'model'
+ if query_params:
+ url = url.copy_with(params=query_params)
+ return str(url)
+
+ async def async_realtime(
+ self,
+ model: str,
+ websocket: Any,
+ logging_obj: LiteLLMLogging,
+ api_base: Optional[str] = None,
+ api_key: Optional[str] = None,
+ client: Optional[Any] = None,
+ timeout: Optional[float] = None,
+ query_params: Optional[RealtimeQueryParams] = None,
+ user_api_key_dict: Optional[Any] = None,
+ litellm_metadata: Optional[dict] = None,
+ **kwargs: Any,
+ ):
+ import websockets
+ from websockets.asyncio.client import ClientConnection
+
+ if api_base is None:
+ api_base = self._get_default_api_base()
+ if api_key is None:
+ raise ValueError("api_key is required for OpenAI realtime calls")
+
+ # Use all query params if provided, else fallback to just model
+ if query_params is None:
+ query_params = {"model": model}
+ url = self._construct_url(api_base, query_params)
+
+ try:
+ # Get provider-specific SSL configuration
+ ssl_config = self._get_ssl_config(url)
+
+ # Get provider-specific headers
+ headers = self._get_additional_headers(api_key)
+
+ # Log a masked request preview consistent with other endpoints.
+ logging_obj.pre_call(
+ input=None,
+ api_key=api_key,
+ additional_args={
+ "api_base": url,
+ "headers": headers,
+ "complete_input_dict": {"query_params": query_params},
+ },
+ )
+ async with websockets.connect( # type: ignore
+ url,
+ additional_headers=headers, # type: ignore
+ max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
+ ssl=ssl_config,
+ ) as backend_ws:
+ realtime_streaming = RealTimeStreaming(
+ websocket,
+ cast(ClientConnection, backend_ws),
+ logging_obj,
+ user_api_key_dict=user_api_key_dict,
+ request_data={"litellm_metadata": litellm_metadata or {}},
+ )
+ await realtime_streaming.bidirectional_forward()
+
+ except websockets.exceptions.InvalidStatusCode as e: # type: ignore
+ await websocket.close(code=e.status_code, reason=str(e))
+ except Exception as e:
+ try:
+ await websocket.close(
+ code=1011, reason=f"Internal server error: {str(e)}"
+ )
+ except RuntimeError as close_error:
+ if "already completed" in str(close_error) or "websocket.close" in str(
+ close_error
+ ):
+ # The WebSocket is already closed or the response is completed, so we can ignore this error
+ pass
+ else:
+ # If it's a different RuntimeError, we might want to log it or handle it differently
+ raise Exception(
+ f"Unexpected error while closing WebSocket: {close_error}"
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/realtime/http_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/realtime/http_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..1663fcd1fcd95724384832f788a37a838d32512f
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/realtime/http_transformation.py
@@ -0,0 +1,54 @@
+"""OpenAI realtime HTTP transformation config (client_secrets + realtime_calls)."""
+
+from typing import Optional
+
+import litellm
+from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
+from litellm.secret_managers.main import get_secret_str
+
+
+class OpenAIRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
+ def get_api_base(self, api_base: Optional[str], **kwargs) -> str:
+ return (
+ api_base
+ or litellm.api_base
+ or get_secret_str("OPENAI_API_BASE")
+ or "https://api.openai.com"
+ )
+
+ def get_api_key(self, api_key: Optional[str], **kwargs) -> str:
+ return (
+ api_key
+ or litellm.api_key
+ or litellm.openai_key
+ or get_secret_str("OPENAI_API_KEY")
+ or ""
+ )
+
+ def get_complete_url(
+ self, api_base: Optional[str], model: str, api_version: Optional[str] = None
+ ) -> str:
+ base = self.get_api_base(api_base).rstrip("/")
+ if base.endswith("/v1"):
+ base = base[:-3]
+ return f"{base}/v1/realtime/client_secrets"
+
+ def get_realtime_calls_url(
+ self, api_base: Optional[str], model: str, api_version: Optional[str] = None
+ ) -> str:
+ base = self.get_api_base(api_base).rstrip("/")
+ if base.endswith("/v1"):
+ base = base[:-3]
+ return f"{base}/v1/realtime/calls"
+
+ def validate_environment(
+ self,
+ headers: dict,
+ model: str,
+ api_key: Optional[str] = None,
+ ) -> dict:
+ return {
+ **headers,
+ "Authorization": f"Bearer {api_key or ''}",
+ "Content-Type": "application/json",
+ }
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8f129a6ff095269ea1a329a6d18340c54f61ba70
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/__init__.py
@@ -0,0 +1,19 @@
+"""
+OpenAI Responses API token counting implementation.
+"""
+
+from litellm.llms.openai.responses.count_tokens.handler import (
+ OpenAICountTokensHandler,
+)
+from litellm.llms.openai.responses.count_tokens.token_counter import (
+ OpenAITokenCounter,
+)
+from litellm.llms.openai.responses.count_tokens.transformation import (
+ OpenAICountTokensConfig,
+)
+
+__all__ = [
+ "OpenAICountTokensHandler",
+ "OpenAICountTokensConfig",
+ "OpenAITokenCounter",
+]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fb5f6dad781c3287d1a5ccd2f9389bbcc677b17
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/handler.py
@@ -0,0 +1,107 @@
+"""
+OpenAI Responses API token counting handler.
+
+Uses httpx for HTTP requests to OpenAI's /v1/responses/input_tokens endpoint.
+"""
+
+import json
+from typing import Any, Dict, List, Optional, Union
+
+import httpx
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+from litellm.llms.openai.common_utils import OpenAIError
+from litellm.llms.openai.responses.count_tokens.transformation import (
+ OpenAICountTokensConfig,
+)
+
+
+class OpenAICountTokensHandler(OpenAICountTokensConfig):
+ """
+ Handler for OpenAI Responses API token counting requests.
+ """
+
+ async def handle_count_tokens_request(
+ self,
+ model: str,
+ input: Union[str, List[Any]],
+ api_key: str,
+ api_base: Optional[str] = None,
+ timeout: Optional[Union[float, httpx.Timeout]] = None,
+ tools: Optional[List[Dict[str, Any]]] = None,
+ instructions: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """
+ Handle a token counting request to OpenAI's Responses API.
+
+ Returns:
+ Dictionary containing {"input_tokens": }
+
+ Raises:
+ OpenAIError: If the API request fails
+ """
+ try:
+ self.validate_request(model, input)
+
+ verbose_logger.debug(
+ f"Processing OpenAI CountTokens request for model: {model}"
+ )
+
+ request_body = self.transform_request_to_count_tokens(
+ model=model,
+ input=input,
+ tools=tools,
+ instructions=instructions,
+ )
+
+ endpoint_url = self.get_openai_count_tokens_endpoint(api_base)
+
+ verbose_logger.debug(f"Making request to: {endpoint_url}")
+
+ headers = self.get_required_headers(api_key)
+
+ async_client = get_async_httpx_client(
+ llm_provider=litellm.LlmProviders.OPENAI
+ )
+
+ request_timeout = (
+ timeout if timeout is not None else litellm.request_timeout
+ )
+
+ response = await async_client.post(
+ endpoint_url,
+ headers=headers,
+ json=request_body,
+ timeout=request_timeout,
+ )
+
+ verbose_logger.debug(f"Response status: {response.status_code}")
+
+ if response.status_code != 200:
+ error_text = response.text
+ verbose_logger.error(f"OpenAI API error: {error_text}")
+ raise OpenAIError(
+ status_code=response.status_code,
+ message=error_text,
+ )
+
+ openai_response = response.json()
+ verbose_logger.debug(f"OpenAI response: {openai_response}")
+ return openai_response
+
+ except OpenAIError:
+ raise
+ except httpx.HTTPStatusError as e:
+ verbose_logger.error(f"HTTP error in CountTokens handler: {str(e)}")
+ raise OpenAIError(
+ status_code=e.response.status_code,
+ message=e.response.text,
+ )
+ except (httpx.RequestError, json.JSONDecodeError, ValueError) as e:
+ verbose_logger.error(f"Error in CountTokens handler: {str(e)}")
+ raise OpenAIError(
+ status_code=500,
+ message=f"CountTokens processing error: {str(e)}",
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/token_counter.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/token_counter.py
new file mode 100644
index 0000000000000000000000000000000000000000..3d3a659075eeda46b12e5c1d29d31c83ba52159a
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/token_counter.py
@@ -0,0 +1,118 @@
+"""
+OpenAI Token Counter implementation using the Responses API /input_tokens endpoint.
+"""
+
+import os
+from typing import Any, Dict, List, Optional
+
+from litellm._logging import verbose_logger
+from litellm.llms.base_llm.base_utils import BaseTokenCounter
+from litellm.llms.openai.common_utils import OpenAIError
+from litellm.llms.openai.responses.count_tokens.handler import (
+ OpenAICountTokensHandler,
+)
+from litellm.llms.openai.responses.count_tokens.transformation import (
+ OpenAICountTokensConfig,
+)
+from litellm.types.utils import LlmProviders, TokenCountResponse
+
+# Global handler instance - reuse across all token counting requests
+openai_count_tokens_handler = OpenAICountTokensHandler()
+
+
+class OpenAITokenCounter(BaseTokenCounter):
+ """Token counter implementation for OpenAI provider using the Responses API."""
+
+ def should_use_token_counting_api(
+ self,
+ custom_llm_provider: Optional[str] = None,
+ ) -> bool:
+ return custom_llm_provider == LlmProviders.OPENAI.value
+
+ async def count_tokens(
+ self,
+ model_to_use: str,
+ messages: Optional[List[Dict[str, Any]]],
+ contents: Optional[List[Dict[str, Any]]],
+ deployment: Optional[Dict[str, Any]] = None,
+ request_model: str = "",
+ tools: Optional[List[Dict[str, Any]]] = None,
+ system: Optional[Any] = None,
+ ) -> Optional[TokenCountResponse]:
+ """
+ Count tokens using OpenAI's Responses API /input_tokens endpoint.
+ """
+ if not messages:
+ return None
+
+ deployment = deployment or {}
+ litellm_params = deployment.get("litellm_params", {})
+
+ # Get OpenAI API key from deployment config or environment
+ api_key = litellm_params.get("api_key")
+ if not api_key:
+ api_key = os.getenv("OPENAI_API_KEY")
+
+ if not api_key:
+ verbose_logger.warning("No OpenAI API key found for token counting")
+ return None
+
+ api_base = litellm_params.get("api_base")
+
+ # Convert chat messages to Responses API input format
+ input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(
+ messages
+ )
+
+ # Use system param if instructions not extracted from messages
+ if instructions is None and system is not None:
+ instructions = system if isinstance(system, str) else str(system)
+
+ # If no input items were produced (e.g., system-only messages), fall back to local counting
+ if not input_items:
+ return None
+
+ try:
+ result = await openai_count_tokens_handler.handle_count_tokens_request(
+ model=model_to_use,
+ input=input_items if input_items is not None else [],
+ api_key=api_key,
+ api_base=api_base,
+ tools=tools,
+ instructions=instructions,
+ )
+
+ if result is not None:
+ return TokenCountResponse(
+ total_tokens=result.get("input_tokens", 0),
+ request_model=request_model,
+ model_used=model_to_use,
+ tokenizer_type="openai_api",
+ original_response=result,
+ )
+ except OpenAIError as e:
+ verbose_logger.warning(
+ f"OpenAI CountTokens API error: status={e.status_code}, message={e.message}"
+ )
+ return TokenCountResponse(
+ total_tokens=0,
+ request_model=request_model,
+ model_used=model_to_use,
+ tokenizer_type="openai_api",
+ error=True,
+ error_message=e.message,
+ status_code=e.status_code,
+ )
+ except Exception as e:
+ verbose_logger.warning(f"Error calling OpenAI CountTokens API: {e}")
+ return TokenCountResponse(
+ total_tokens=0,
+ request_model=request_model,
+ model_used=model_to_use,
+ tokenizer_type="openai_api",
+ error=True,
+ error_message=str(e),
+ status_code=500,
+ )
+
+ return None
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..41d1a01ec66df4ee1e668b061daca86db0982e37
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/count_tokens/transformation.py
@@ -0,0 +1,160 @@
+"""
+OpenAI Responses API token counting transformation logic.
+
+This module handles the transformation of requests to OpenAI's /v1/responses/input_tokens endpoint.
+"""
+
+from typing import Any, Dict, List, Optional, Union
+
+
+class OpenAICountTokensConfig:
+ """
+ Configuration and transformation logic for OpenAI Responses API token counting.
+
+ OpenAI Responses API Token Counting Specification:
+ - Endpoint: POST https://api.openai.com/v1/responses/input_tokens
+ - Response: {"input_tokens": }
+ """
+
+ def get_openai_count_tokens_endpoint(self, api_base: Optional[str] = None) -> str:
+ base = api_base or "https://api.openai.com/v1"
+ base = base.rstrip("/")
+ return f"{base}/responses/input_tokens"
+
+ def transform_request_to_count_tokens(
+ self,
+ model: str,
+ input: Union[str, List[Any]],
+ tools: Optional[List[Dict[str, Any]]] = None,
+ instructions: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ """
+ Transform request to OpenAI Responses API token counting format.
+
+ The Responses API uses `input` (not `messages`) and `instructions` (not `system`).
+ """
+ request: Dict[str, Any] = {
+ "model": model,
+ "input": input,
+ }
+
+ if instructions is not None:
+ request["instructions"] = instructions
+
+ if tools is not None:
+ request["tools"] = self._transform_tools_for_responses_api(tools)
+
+ return request
+
+ def get_required_headers(self, api_key: str) -> Dict[str, str]:
+ return {
+ "Content-Type": "application/json",
+ "Authorization": f"Bearer {api_key}",
+ }
+
+ def validate_request(self, model: str, input: Union[str, List[Any]]) -> None:
+ if not model:
+ raise ValueError("model parameter is required")
+
+ if not input:
+ raise ValueError("input parameter is required")
+
+ @staticmethod
+ def _transform_tools_for_responses_api(
+ tools: List[Dict[str, Any]],
+ ) -> List[Dict[str, Any]]:
+ """
+ Transform OpenAI chat tools format to Responses API tools format.
+
+ Chat format: {"type": "function", "function": {"name": "...", "parameters": {...}}}
+ Responses format: {"type": "function", "name": "...", "parameters": {...}}
+ """
+ transformed = []
+ for tool in tools:
+ if tool.get("type") == "function" and "function" in tool:
+ func = tool["function"]
+ item: Dict[str, Any] = {
+ "type": "function",
+ "name": func.get("name", ""),
+ "description": func.get("description", ""),
+ "parameters": func.get("parameters", {}),
+ }
+ if "strict" in func:
+ item["strict"] = func["strict"]
+ transformed.append(item)
+ else:
+ # Pass through non-function tools (e.g., web_search, file_search)
+ transformed.append(tool)
+ return transformed
+
+ @staticmethod
+ def messages_to_responses_input(
+ messages: List[Dict[str, Any]],
+ ) -> tuple:
+ """
+ Convert standard chat messages format to OpenAI Responses API input format.
+
+ Returns:
+ (input_items, instructions) tuple where instructions is extracted
+ from system/developer messages.
+ """
+ input_items: List[Dict[str, Any]] = []
+ instructions_parts: List[str] = []
+
+ for msg in messages:
+ role = msg.get("role", "")
+ content = msg.get("content") or ""
+
+ if role in ("system", "developer"):
+ # Extract system/developer messages as instructions
+ if isinstance(content, str):
+ instructions_parts.append(content)
+ elif isinstance(content, list):
+ # Handle content blocks - extract text
+ text_parts = []
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ text_parts.append(block.get("text", ""))
+ elif isinstance(block, str):
+ text_parts.append(block)
+ instructions_parts.append("\n".join(text_parts))
+ elif role == "user":
+ if isinstance(content, list):
+ # Extract text from content blocks for Responses API
+ text_parts = []
+ for block in content:
+ if isinstance(block, dict) and block.get("type") == "text":
+ text_parts.append(block.get("text", ""))
+ elif isinstance(block, str):
+ text_parts.append(block)
+ content = "\n".join(text_parts)
+ input_items.append({"role": "user", "content": content})
+ elif role == "assistant":
+ # Map tool_calls to Responses API function_call items
+ tool_calls = msg.get("tool_calls")
+ if content:
+ input_items.append({"role": "assistant", "content": content})
+ if tool_calls:
+ for tc in tool_calls:
+ func = tc.get("function", {})
+ input_items.append(
+ {
+ "type": "function_call",
+ "call_id": tc.get("id", ""),
+ "name": func.get("name", ""),
+ "arguments": func.get("arguments", ""),
+ }
+ )
+ elif not content:
+ input_items.append({"role": "assistant", "content": content})
+ elif role == "tool":
+ input_items.append(
+ {
+ "type": "function_call_output",
+ "call_id": msg.get("tool_call_id", ""),
+ "output": content if isinstance(content, str) else str(content),
+ }
+ )
+
+ instructions = "\n".join(instructions_parts) if instructions_parts else None
+ return input_items, instructions
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/guardrail_translation/README.md b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/guardrail_translation/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..bc1bd6f4f2ce2925c55e91eba8f03d61446f5b7c
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/guardrail_translation/README.md
@@ -0,0 +1,119 @@
+# OpenAI Responses API Guardrail Translation Handler
+
+This module provides guardrail translation support for the OpenAI Responses API format.
+
+## Overview
+
+The `OpenAIResponsesHandler` class handles the translation of guardrail operations for both input and output of the Responses API. It follows the same pattern as the Chat Completions handler but is adapted for the Responses API's specific data structures.
+
+## Responses API Format
+
+### Input Format
+The Responses API accepts input in two formats:
+
+1. **String input**: Simple text string
+ ```python
+ {"input": "Hello world", "model": "gpt-4"}
+ ```
+
+2. **List input**: Array of message objects (ResponseInputParam)
+ ```python
+ {
+ "input": [
+ {
+ "role": "user",
+ "content": "Hello", # Can be string or list of content items
+ "type": "message"
+ }
+ ],
+ "model": "gpt-4"
+ }
+ ```
+
+### Output Format
+The Responses API returns a `ResponsesAPIResponse` object with:
+
+```python
+{
+ "id": "resp_123",
+ "output": [
+ {
+ "type": "message",
+ "id": "msg_123",
+ "status": "completed",
+ "role": "assistant",
+ "content": [
+ {
+ "type": "output_text",
+ "text": "Assistant response",
+ "annotations": []
+ }
+ ]
+ }
+ ]
+}
+```
+
+## Usage
+
+The handler is automatically discovered and registered for `CallTypes.responses` and `CallTypes.aresponses`.
+
+### Example
+
+```python
+from litellm.llms import get_guardrail_translation_mapping
+from litellm.types.utils import CallTypes
+
+# Get the handler
+handler_class = get_guardrail_translation_mapping(CallTypes.responses)
+handler = handler_class()
+
+# Process input
+data = {"input": "User message", "model": "gpt-4"}
+processed_data = await handler.process_input_messages(data, guardrail_instance)
+
+# Process output
+response = await litellm.aresponses(**processed_data)
+processed_response = await handler.process_output_response(response, guardrail_instance)
+```
+
+## Key Methods
+
+### `process_input_messages(data, guardrail_to_apply)`
+Processes input data by:
+1. Handling both string and list input formats
+2. Extracting text content from messages
+3. Applying guardrails to text content in parallel
+4. Mapping guardrail responses back to the original structure
+
+### `process_output_response(response, guardrail_to_apply)`
+Processes output response by:
+1. Extracting text from output items' content
+2. Applying guardrails to all text content in parallel
+3. Replacing original text with guardrailed versions
+
+## Extending the Handler
+
+The handler can be customized by overriding these methods:
+
+- `_extract_input_text_and_create_tasks()`: Customize input text extraction logic
+- `_apply_guardrail_responses_to_input()`: Customize how guardrail responses are applied to input
+- `_extract_output_text_and_create_tasks()`: Customize output text extraction logic
+- `_apply_guardrail_responses_to_output()`: Customize how guardrail responses are applied to output
+- `_has_text_content()`: Customize text content detection
+
+## Testing
+
+Comprehensive tests are available in `tests/llm_translation/test_openai_responses_guardrail_handler.py`:
+
+```bash
+pytest tests/llm_translation/test_openai_responses_guardrail_handler.py -v
+```
+
+## Implementation Details
+
+- **Parallel Processing**: All text content is processed in parallel using `asyncio.gather()`
+- **Mapping Tracking**: Uses tuples to track the location of each text segment for accurate replacement
+- **Type Safety**: Handles both Pydantic objects and dict representations
+- **Multimodal Support**: Properly handles mixed content with text and other media types
+
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/guardrail_translation/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/guardrail_translation/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2d9e5375c1d2ade24dd16fdd7ac6d91fad0ffcb
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/guardrail_translation/__init__.py
@@ -0,0 +1,12 @@
+"""OpenAI Responses API handler for Unified Guardrails."""
+
+from litellm.llms.openai.responses.guardrail_translation.handler import (
+ OpenAIResponsesHandler,
+)
+from litellm.types.utils import CallTypes
+
+guardrail_translation_mappings = {
+ CallTypes.responses: OpenAIResponsesHandler,
+ CallTypes.aresponses: OpenAIResponsesHandler,
+}
+__all__ = ["guardrail_translation_mappings"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/guardrail_translation/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/guardrail_translation/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..466e2e76f1838f3b077cce0c1ea48a9a860bf8ad
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -0,0 +1,760 @@
+"""
+OpenAI Responses API Handler for Unified Guardrails
+
+This module provides a class-based handler for OpenAI Responses API format.
+The class methods can be overridden for custom behavior.
+
+Pattern Overview:
+-----------------
+1. Extract text content from input/output (both string and list formats)
+2. Create async tasks to apply guardrails to each text segment
+3. Track mappings to know where each response belongs
+4. Apply guardrail responses back to the original structure
+
+Responses API Format:
+---------------------
+Input: Union[str, List[Dict]] where each dict has:
+ - role: str
+ - content: Union[str, List[Dict]] (can have text items)
+ - type: str (e.g., "message")
+
+Output: response.output is List[GenericResponseOutputItem] where each has:
+ - type: str (e.g., "message")
+ - id: str
+ - status: str
+ - role: str
+ - content: List[OutputText] where OutputText has:
+ - type: str (e.g., "output_text")
+ - text: str
+"""
+
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
+
+from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
+from pydantic import BaseModel
+
+from litellm._logging import verbose_proxy_logger
+from litellm.completion_extras.litellm_responses_transformation.transformation import (
+ LiteLLMResponsesTransformationHandler,
+ OpenAiResponsesToChatCompletionStreamIterator,
+)
+from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
+from litellm.responses.litellm_completion_transformation.transformation import (
+ LiteLLMCompletionResponsesConfig,
+)
+from litellm.types.llms.openai import (
+ ChatCompletionToolCallChunk,
+ ChatCompletionToolParam,
+)
+from litellm.types.responses.main import (
+ GenericResponseOutputItem,
+ OutputFunctionToolCall,
+ OutputText,
+)
+from litellm.types.utils import GenericGuardrailAPIInputs
+
+if TYPE_CHECKING:
+ from litellm.integrations.custom_guardrail import CustomGuardrail
+ from litellm.types.llms.openai import ResponseInputParam
+ from litellm.types.utils import ResponsesAPIResponse
+
+
+class OpenAIResponsesHandler(BaseTranslation):
+ """
+ Handler for processing OpenAI Responses API with guardrails.
+
+ This class provides methods to:
+ 1. Process input (pre-call hook)
+ 2. Process output response (post-call hook)
+
+ Methods can be overridden to customize behavior for different message formats.
+ """
+
+ async def process_input_messages(
+ self,
+ data: dict,
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process input by applying guardrails to text content.
+
+ Handles both string input and list of message objects.
+ """
+ input_data: Optional[Union[str, "ResponseInputParam"]] = data.get("input")
+ tools_to_check: List[ChatCompletionToolParam] = []
+ if input_data is None:
+ return data
+
+ structured_messages = (
+ LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+ input=input_data,
+ responses_api_request=data,
+ )
+ )
+
+ # Handle simple string input
+ if isinstance(input_data, str):
+ inputs = GenericGuardrailAPIInputs(texts=[input_data])
+ original_tools: List[Dict[str, Any]] = []
+
+ # Extract and transform tools if present
+ if "tools" in data and data["tools"]:
+ original_tools = list(data["tools"])
+ self._extract_and_transform_tools(data["tools"], tools_to_check)
+ if tools_to_check:
+ inputs["tools"] = tools_to_check
+ if structured_messages:
+ inputs["structured_messages"] = structured_messages # type: ignore
+ # Include model information if available
+ model = data.get("model")
+ if model:
+ inputs["model"] = model
+
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=data,
+ input_type="request",
+ logging_obj=litellm_logging_obj,
+ )
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+ data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data
+ self._apply_guardrailed_tools_to_data(
+ data, original_tools, guardrailed_inputs.get("tools")
+ )
+ verbose_proxy_logger.debug("OpenAI Responses API: Processed string input")
+ return data
+
+ # Handle list input (ResponseInputParam)
+ if not isinstance(input_data, list):
+ return data
+
+ texts_to_check: List[str] = []
+ images_to_check: List[str] = []
+ task_mappings: List[Tuple[int, Optional[int]]] = []
+ original_tools_list: List[Dict[str, Any]] = list(data.get("tools") or [])
+
+ # Step 1: Extract all text content, images, and tools
+ for msg_idx, message in enumerate(input_data):
+ self._extract_input_text_and_images(
+ message=message,
+ msg_idx=msg_idx,
+ texts_to_check=texts_to_check,
+ images_to_check=images_to_check,
+ task_mappings=task_mappings,
+ )
+
+ # Extract and transform tools if present
+ if "tools" in data and data["tools"]:
+ self._extract_and_transform_tools(data["tools"], tools_to_check)
+
+ # Step 2: Apply guardrail to all texts in batch
+ if texts_to_check:
+ inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
+ if images_to_check:
+ inputs["images"] = images_to_check
+ if tools_to_check:
+ inputs["tools"] = tools_to_check
+ if structured_messages:
+ inputs["structured_messages"] = structured_messages # type: ignore
+ # Include model information if available
+ model = data.get("model")
+ if model:
+ inputs["model"] = model
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=data,
+ input_type="request",
+ logging_obj=litellm_logging_obj,
+ )
+
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+ self._apply_guardrailed_tools_to_data(
+ data,
+ original_tools_list,
+ guardrailed_inputs.get("tools"),
+ )
+
+ # Step 3: Map guardrail responses back to original input structure
+ await self._apply_guardrail_responses_to_input(
+ messages=input_data,
+ responses=guardrailed_texts,
+ task_mappings=task_mappings,
+ )
+
+ verbose_proxy_logger.debug(
+ "OpenAI Responses API: Processed input messages: %s", input_data
+ )
+
+ return data
+
+ def extract_request_tool_names(self, data: dict) -> List[str]:
+ """Extract tool names from Responses API request (tools[].name for function, tools[].server_label for mcp)."""
+ names: List[str] = []
+ for tool in data.get("tools") or []:
+ if not isinstance(tool, dict):
+ continue
+ if tool.get("type") == "function" and tool.get("name"):
+ names.append(str(tool["name"]))
+ elif tool.get("type") == "mcp" and tool.get("server_label"):
+ names.append(str(tool["server_label"]))
+ return names
+
+ def _extract_and_transform_tools(
+ self,
+ tools: List[Dict[str, Any]],
+ tools_to_check: List[ChatCompletionToolParam],
+ ) -> None:
+ """
+ Extract and transform tools from Responses API format to Chat Completion format.
+
+ Uses the LiteLLM transformation function to convert Responses API tools
+ to Chat Completion tools that can be passed to guardrails.
+ """
+ if tools is not None and isinstance(tools, list):
+ # Transform Responses API tools to Chat Completion tools
+ (
+ transformed_tools,
+ _,
+ ) = LiteLLMCompletionResponsesConfig.transform_responses_api_tools_to_chat_completion_tools(
+ tools # type: ignore
+ )
+ tools_to_check.extend(
+ cast(List[ChatCompletionToolParam], transformed_tools)
+ )
+
+ def _remap_tools_to_responses_api_format(
+ self, guardrailed_tools: List[Any]
+ ) -> List[Dict[str, Any]]:
+ """
+ Remap guardrail-returned tools (Chat Completion format) back to
+ Responses API request tool format.
+ """
+ return LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools(
+ guardrailed_tools # type: ignore
+ )
+
+ def _merge_tools_after_guardrail(
+ self,
+ original_tools: List[Dict[str, Any]],
+ remapped: List[Dict[str, Any]],
+ ) -> List[Dict[str, Any]]:
+ """
+ Merge remapped guardrailed tools with original tools that were not sent
+ to the guardrail (e.g. web_search, web_search_preview), preserving order.
+ """
+ if not original_tools:
+ return remapped
+ result: List[Dict[str, Any]] = []
+ j = 0
+ for tool in original_tools:
+ if isinstance(tool, dict) and tool.get("type") in (
+ "web_search",
+ "web_search_preview",
+ ):
+ result.append(tool)
+ else:
+ if j < len(remapped):
+ result.append(remapped[j])
+ j += 1
+ return result
+
+ def _apply_guardrailed_tools_to_data(
+ self,
+ data: dict,
+ original_tools: List[Dict[str, Any]],
+ guardrailed_tools: Optional[List[Any]],
+ ) -> None:
+ """Remap guardrailed tools to Responses API format and merge with original, then set data['tools']."""
+ if guardrailed_tools is not None:
+ remapped = self._remap_tools_to_responses_api_format(guardrailed_tools)
+ data["tools"] = self._merge_tools_after_guardrail(original_tools, remapped)
+
+ def _extract_input_text_and_images(
+ self,
+ message: Any, # Can be Dict[str, Any] or ResponseInputParam
+ msg_idx: int,
+ texts_to_check: List[str],
+ images_to_check: List[str],
+ task_mappings: List[Tuple[int, Optional[int]]],
+ ) -> None:
+ """
+ Extract text content and images from an input message.
+
+ Override this method to customize text/image extraction logic.
+ """
+ content = message.get("content", None)
+ if content is None:
+ return
+
+ if isinstance(content, str):
+ # Simple string content
+ texts_to_check.append(content)
+ task_mappings.append((msg_idx, None))
+
+ elif isinstance(content, list):
+ # List content (e.g., multimodal with text and images)
+ for content_idx, content_item in enumerate(content):
+ if isinstance(content_item, dict):
+ # Extract text
+ text_str = content_item.get("text", None)
+ if text_str is not None:
+ texts_to_check.append(text_str)
+ task_mappings.append((msg_idx, int(content_idx)))
+
+ # Extract images
+ if content_item.get("type") == "image_url":
+ image_url = content_item.get("image_url", {})
+ if isinstance(image_url, dict):
+ url = image_url.get("url")
+ if url:
+ images_to_check.append(url)
+
+ async def _apply_guardrail_responses_to_input(
+ self,
+ messages: Any, # Can be List[Dict[str, Any]] or ResponseInputParam
+ responses: List[str],
+ task_mappings: List[Tuple[int, Optional[int]]],
+ ) -> None:
+ """
+ Apply guardrail responses back to input messages.
+
+ Override this method to customize how responses are applied.
+ """
+ for task_idx, guardrail_response in enumerate(responses):
+ mapping = task_mappings[task_idx]
+ msg_idx = cast(int, mapping[0])
+ content_idx_optional = cast(Optional[int], mapping[1])
+
+ content = messages[msg_idx].get("content", None)
+ if content is None:
+ continue
+
+ if isinstance(content, str) and content_idx_optional is None:
+ # Replace string content with guardrail response
+ messages[msg_idx]["content"] = guardrail_response
+
+ elif isinstance(content, list) and content_idx_optional is not None:
+ # Replace specific text item in list content
+ if isinstance(messages[msg_idx]["content"][content_idx_optional], dict):
+ messages[msg_idx]["content"][content_idx_optional][
+ "text"
+ ] = guardrail_response
+
+ async def process_output_response(
+ self,
+ response: "ResponsesAPIResponse",
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ user_api_key_dict: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process output response by applying guardrails to text content and tool calls.
+
+ Args:
+ response: LiteLLM ResponsesAPIResponse object
+ guardrail_to_apply: The guardrail instance to apply
+ litellm_logging_obj: Optional logging object
+ user_api_key_dict: User API key metadata to pass to guardrails
+
+ Returns:
+ Modified response with guardrail applied to content
+
+ Response Format Support:
+ - response.output is a list of output items
+ - Each output item can be:
+ * GenericResponseOutputItem with a content list of OutputText objects
+ * ResponseFunctionToolCall with tool call data
+ - Each OutputText object has a text field
+ """
+
+ texts_to_check: List[str] = []
+ images_to_check: List[str] = []
+ tool_calls_to_check: List[ChatCompletionToolCallChunk] = []
+ task_mappings: List[Tuple[int, int]] = []
+ # Track (output_item_index, content_index) for each text
+
+ # Handle both dict and Pydantic object responses
+ if isinstance(response, dict):
+ response_output = response.get("output", [])
+ elif hasattr(response, "output"):
+ response_output = response.output or []
+ else:
+ verbose_proxy_logger.debug(
+ "OpenAI Responses API: No output found in response"
+ )
+ return response
+
+ if not response_output:
+ verbose_proxy_logger.debug("OpenAI Responses API: Empty output in response")
+ return response
+
+ # Step 1: Extract all text content and tool calls from response output
+ for output_idx, output_item in enumerate(response_output):
+ self._extract_output_text_and_images(
+ output_item=output_item,
+ output_idx=output_idx,
+ texts_to_check=texts_to_check,
+ images_to_check=images_to_check,
+ task_mappings=task_mappings,
+ tool_calls_to_check=tool_calls_to_check,
+ )
+
+ # Step 2: Apply guardrail to all texts in batch
+ if texts_to_check or tool_calls_to_check:
+ # Create a request_data dict with response info and user API key metadata
+ request_data: dict = {"response": response}
+
+ # Add user API key metadata with prefixed keys
+ user_metadata = self.transform_user_api_key_dict_to_metadata(
+ user_api_key_dict
+ )
+ if user_metadata:
+ request_data["litellm_metadata"] = user_metadata
+
+ inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
+ if images_to_check:
+ inputs["images"] = images_to_check
+ if tool_calls_to_check:
+ inputs["tool_calls"] = tool_calls_to_check
+ # Include model information from the response if available
+ response_model = None
+ if isinstance(response, dict):
+ response_model = response.get("model")
+ elif hasattr(response, "model"):
+ response_model = getattr(response, "model", None)
+ if response_model:
+ inputs["model"] = response_model
+
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=request_data,
+ input_type="response",
+ logging_obj=litellm_logging_obj,
+ )
+
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+
+ # Step 3: Map guardrail responses back to original response structure
+ await self._apply_guardrail_responses_to_output(
+ response=response,
+ responses=guardrailed_texts,
+ task_mappings=task_mappings,
+ )
+
+ verbose_proxy_logger.debug(
+ "OpenAI Responses API: Processed output response: %s", response
+ )
+
+ return response
+
+ async def process_output_streaming_response(
+ self,
+ responses_so_far: List[Any],
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ user_api_key_dict: Optional[Any] = None,
+ ) -> List[Any]:
+ """
+ Process output streaming response by applying guardrails to text content.
+ """
+
+ final_chunk = responses_so_far[-1]
+
+ if final_chunk.get("type") == "response.output_item.done":
+ # convert openai response to model response
+ model_response_stream = OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
+ final_chunk
+ )
+
+ tool_calls = model_response_stream.choices[0].delta.tool_calls
+ if tool_calls:
+ inputs = GenericGuardrailAPIInputs()
+ inputs["tool_calls"] = cast(
+ List[ChatCompletionToolCallChunk], tool_calls
+ )
+ # Include model information if available
+ if (
+ hasattr(model_response_stream, "model")
+ and model_response_stream.model
+ ):
+ inputs["model"] = model_response_stream.model
+ _guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data={},
+ input_type="response",
+ logging_obj=litellm_logging_obj,
+ )
+ return responses_so_far
+ elif final_chunk.get("type") == "response.completed":
+ # convert openai response to model response
+ outputs = final_chunk.get("response", {}).get("output", [])
+
+ model_response_choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices(
+ output_items=outputs,
+ handle_raw_dict_callback=None,
+ )
+
+ if model_response_choices:
+ tool_calls = model_response_choices[0].message.tool_calls
+ text = model_response_choices[0].message.content
+ guardrail_inputs = GenericGuardrailAPIInputs()
+ if text:
+ guardrail_inputs["texts"] = [text]
+ if tool_calls:
+ guardrail_inputs["tool_calls"] = cast(
+ List[ChatCompletionToolCallChunk], tool_calls
+ )
+ # Include model information from the response if available
+ response_model = final_chunk.get("response", {}).get("model")
+ if response_model:
+ guardrail_inputs["model"] = response_model
+ if tool_calls or text:
+ _guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=guardrail_inputs,
+ request_data={},
+ input_type="response",
+ logging_obj=litellm_logging_obj,
+ )
+ return responses_so_far
+ else:
+ verbose_proxy_logger.debug(
+ "Skipping output guardrail - model response has no choices"
+ )
+ # model_response_stream = OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(final_chunk)
+ # tool_calls = model_response_stream.choices[0].tool_calls
+ # convert openai response to model response
+ string_so_far = self.get_streaming_string_so_far(responses_so_far)
+ inputs = GenericGuardrailAPIInputs(texts=[string_so_far])
+ # Try to get model from the final chunk if available
+ if isinstance(final_chunk, dict):
+ response_model = (
+ final_chunk.get("response", {}).get("model")
+ if isinstance(final_chunk.get("response"), dict)
+ else None
+ )
+ if response_model:
+ inputs["model"] = response_model
+ _guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data={},
+ input_type="response",
+ logging_obj=litellm_logging_obj,
+ )
+ return responses_so_far
+
+ def _check_streaming_has_ended(self, responses_so_far: List[Any]) -> bool:
+ """
+ Check if the streaming has ended.
+ """
+ return all(
+ response.choices[0].finish_reason is not None
+ for response in responses_so_far
+ )
+
+ def get_streaming_string_so_far(self, responses_so_far: List[Any]) -> str:
+ """
+ Get the string so far from the responses so far.
+ """
+ return "".join([response.get("text", "") for response in responses_so_far])
+
+ def _has_text_content(self, response: "ResponsesAPIResponse") -> bool:
+ """
+ Check if response has any text content to process.
+
+ Override this method to customize text content detection.
+ """
+ if not hasattr(response, "output") or response.output is None:
+ return False
+
+ for output_item in response.output:
+ if isinstance(output_item, BaseModel):
+ try:
+ generic_response_output_item = (
+ GenericResponseOutputItem.model_validate(
+ output_item.model_dump()
+ )
+ )
+ if generic_response_output_item.content:
+ output_item = generic_response_output_item
+ except Exception:
+ continue
+ if isinstance(output_item, (GenericResponseOutputItem, dict)):
+ content = (
+ output_item.content
+ if isinstance(output_item, GenericResponseOutputItem)
+ else output_item.get("content", [])
+ )
+ if content:
+ for content_item in content:
+ # Check if it's an OutputText with text
+ if isinstance(content_item, OutputText):
+ if content_item.text:
+ return True
+ elif isinstance(content_item, dict):
+ if content_item.get("text"):
+ return True
+ return False
+
+ def _extract_output_text_and_images(
+ self,
+ output_item: Any,
+ output_idx: int,
+ texts_to_check: List[str],
+ images_to_check: List[str],
+ task_mappings: List[Tuple[int, int]],
+ tool_calls_to_check: Optional[List[ChatCompletionToolCallChunk]] = None,
+ ) -> None:
+ """
+ Extract text content, images, and tool calls from a response output item.
+
+ Override this method to customize text/image/tool extraction logic.
+ """
+
+ # Check if this is a tool call (OutputFunctionToolCall)
+ if isinstance(output_item, OutputFunctionToolCall):
+ if tool_calls_to_check is not None:
+ tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call(
+ tool_call_item=output_item,
+ index=output_idx,
+ )
+ tool_calls_to_check.append(
+ cast(ChatCompletionToolCallChunk, tool_call_dict)
+ )
+ return
+ elif (
+ isinstance(output_item, BaseModel)
+ and hasattr(output_item, "type")
+ and getattr(output_item, "type") == "function_call"
+ ):
+ if tool_calls_to_check is not None:
+ tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call(
+ tool_call_item=output_item,
+ index=output_idx,
+ )
+ tool_calls_to_check.append(
+ cast(ChatCompletionToolCallChunk, tool_call_dict)
+ )
+ return
+ elif (
+ isinstance(output_item, dict) and output_item.get("type") == "function_call"
+ ):
+ # Handle dict representation of tool call
+ if tool_calls_to_check is not None:
+ # Convert dict to ResponseFunctionToolCall for processing
+ try:
+ tool_call_obj = ResponseFunctionToolCall(**output_item)
+ tool_call_dict = LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call(
+ tool_call_item=tool_call_obj,
+ index=output_idx,
+ )
+ tool_calls_to_check.append(
+ cast(ChatCompletionToolCallChunk, tool_call_dict)
+ )
+ except Exception:
+ pass
+ return
+
+ # Handle both GenericResponseOutputItem and dict
+ content: Optional[Union[List[OutputText], List[dict]]] = None
+ if isinstance(output_item, BaseModel):
+ try:
+ output_item_dump = output_item.model_dump()
+ generic_response_output_item = GenericResponseOutputItem.model_validate(
+ output_item_dump
+ )
+ if generic_response_output_item.content:
+ content = generic_response_output_item.content
+ except Exception:
+ # Try to extract content directly from output_item if validation fails
+ if hasattr(output_item, "content") and output_item.content: # type: ignore
+ content = output_item.content # type: ignore
+ else:
+ return
+ elif isinstance(output_item, dict):
+ content = output_item.get("content", [])
+ else:
+ return
+
+ if not content:
+ return
+
+ verbose_proxy_logger.debug(
+ "OpenAI Responses API: Processing output item: %s", output_item
+ )
+
+ # Iterate through content items (list of OutputText objects)
+ for content_idx, content_item in enumerate(content):
+ # Handle both OutputText objects and dicts
+ if isinstance(content_item, OutputText):
+ text_content = content_item.text
+ elif isinstance(content_item, dict):
+ text_content = content_item.get("text")
+ else:
+ continue
+
+ if text_content:
+ texts_to_check.append(text_content)
+ task_mappings.append((output_idx, int(content_idx)))
+
+ async def _apply_guardrail_responses_to_output(
+ self,
+ response: "ResponsesAPIResponse",
+ responses: List[str],
+ task_mappings: List[Tuple[int, int]],
+ ) -> None:
+ """
+ Apply guardrail responses back to output response.
+
+ Override this method to customize how responses are applied.
+ """
+ # Handle both dict and Pydantic object responses
+ if isinstance(response, dict):
+ response_output = response.get("output", [])
+ elif hasattr(response, "output"):
+ response_output = response.output or []
+ else:
+ return
+
+ for task_idx, guardrail_response in enumerate(responses):
+ mapping = task_mappings[task_idx]
+ output_idx = cast(int, mapping[0])
+ content_idx = cast(int, mapping[1])
+
+ if output_idx >= len(response_output):
+ continue
+
+ output_item = response_output[output_idx]
+
+ # Handle both GenericResponseOutputItem, BaseModel, and dict
+ if isinstance(output_item, GenericResponseOutputItem):
+ if output_item.content and content_idx < len(output_item.content):
+ content_item = output_item.content[content_idx]
+ if isinstance(content_item, OutputText):
+ content_item.text = guardrail_response
+ elif isinstance(content_item, dict):
+ content_item["text"] = guardrail_response
+ elif isinstance(output_item, BaseModel):
+ # Handle other Pydantic models by converting to GenericResponseOutputItem
+ try:
+ generic_item = GenericResponseOutputItem.model_validate(
+ output_item.model_dump()
+ )
+ if generic_item.content and content_idx < len(generic_item.content):
+ content_item = generic_item.content[content_idx]
+ if isinstance(content_item, OutputText):
+ content_item.text = guardrail_response
+ # Update the original response output
+ if hasattr(output_item, "content") and output_item.content: # type: ignore
+ original_content = output_item.content[content_idx] # type: ignore
+ if hasattr(original_content, "text"):
+ original_content.text = guardrail_response # type: ignore
+ except Exception:
+ pass
+ elif isinstance(output_item, dict):
+ content = output_item.get("content", [])
+ if content and content_idx < len(content):
+ if isinstance(content[content_idx], dict):
+ content[content_idx]["text"] = guardrail_response
+ elif hasattr(content[content_idx], "text"):
+ content[content_idx].text = guardrail_response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..cafb745862ded74296acab8f20f11b17d8fdceb0
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/responses/transformation.py
@@ -0,0 +1,583 @@
+from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast, get_type_hints
+
+import httpx
+from openai.types.responses import ResponseReasoningItem
+from pydantic import BaseModel, ValidationError
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import process_response_headers
+from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+ _safe_convert_created_field,
+)
+from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import *
+from litellm.types.responses.main import *
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import LlmProviders
+
+from ..common_utils import OpenAIError
+
+if TYPE_CHECKING:
+ from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+ LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+ LiteLLMLoggingObj = Any
+
+
+class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
+ @property
+ def custom_llm_provider(self) -> LlmProviders:
+ return LlmProviders.OPENAI
+
+ def supports_native_file_search(self) -> bool:
+ return True
+
+ def get_supported_openai_params(self, model: str) -> list:
+ """
+ All OpenAI Responses API params are supported
+ """
+ supported_params = get_type_hints(ResponsesAPIRequestParams).keys()
+ return list(
+ set(
+ [
+ "input",
+ "model",
+ "extra_headers",
+ "extra_query",
+ "extra_body",
+ "timeout",
+ ]
+ + list(supported_params)
+ )
+ )
+
+ def map_openai_params(
+ self,
+ response_api_optional_params: ResponsesAPIOptionalRequestParams,
+ model: str,
+ drop_params: bool,
+ ) -> Dict:
+ """No mapping applied since inputs are in OpenAI spec already"""
+ return dict(response_api_optional_params)
+
+ def transform_responses_api_request(
+ self,
+ model: str,
+ input: Union[str, ResponseInputParam],
+ response_api_optional_request_params: Dict,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Dict:
+ """No transform applied since inputs are in OpenAI spec already"""
+
+ input = self._validate_input_param(input)
+ final_request_params = dict(
+ ResponsesAPIRequestParams(
+ model=model, input=input, **response_api_optional_request_params
+ )
+ )
+
+ return final_request_params
+
+ def _validate_input_param(
+ self, input: Union[str, ResponseInputParam]
+ ) -> Union[str, ResponseInputParam]:
+ """
+ Ensure all input fields if pydantic are converted to dict
+
+ OpenAI API Fails when we try to JSON dumps specific input pydantic fields.
+ This function ensures all input fields are converted to dict.
+ """
+ if isinstance(input, list):
+ validated_input = []
+ for item in input:
+ # if it's pydantic, convert to dict
+ if isinstance(item, BaseModel):
+ validated_input.append(item.model_dump(exclude_none=True))
+ elif isinstance(item, dict):
+ # Handle reasoning items specifically to filter out status=None
+ if item.get("type") == "reasoning":
+ verbose_logger.debug(f"Handling reasoning item: {item}")
+ # Type assertion since we know it's a dict at this point
+ dict_item = cast(Dict[str, Any], item)
+ filtered_item = self._handle_reasoning_item(dict_item)
+ else:
+ # For other dict items, just pass through
+ filtered_item = cast(Dict[str, Any], item)
+ validated_input.append(filtered_item)
+ else:
+ validated_input.append(item)
+ return validated_input # type: ignore
+ # Input is expected to be either str or List, no single BaseModel expected
+ return input
+
+ def _handle_reasoning_item(self, item: Dict[str, Any]) -> Dict[str, Any]:
+ """
+ Handle reasoning items specifically to filter out status=None using OpenAI's model.
+ Issue: https://github.com/BerriAI/litellm/issues/13484
+ OpenAI API does not accept ReasoningItem(status=None), so we need to:
+ 1. Check if the item is a reasoning type
+ 2. Create a ResponseReasoningItem object with the item data
+ 3. Convert it back to dict with exclude_none=True to filter None values
+ """
+ if item.get("type") == "reasoning":
+ try:
+ # Ensure required fields are present for ResponseReasoningItem
+ item_data = dict(item)
+ if "summary" not in item_data:
+ item_data["summary"] = (
+ item_data.get("reasoning_content", "")[:100] + "..."
+ if len(item_data.get("reasoning_content", "")) > 100
+ else item_data.get("reasoning_content", "")
+ )
+
+ # Create ResponseReasoningItem object from the item data
+ reasoning_item = ResponseReasoningItem(**item_data)
+
+ # Convert back to dict with exclude_none=True to exclude None fields
+ dict_reasoning_item = reasoning_item.model_dump(exclude_none=True)
+
+ return dict_reasoning_item
+ except Exception as e:
+ verbose_logger.debug(
+ f"Failed to create ResponseReasoningItem, falling back to manual filtering: {e}"
+ )
+ # Fallback: manually filter out known None fields
+ filtered_item = {
+ k: v
+ for k, v in item.items()
+ if v is not None
+ or k not in {"status", "content", "encrypted_content"}
+ }
+ return filtered_item
+ return item
+
+ def transform_response_api_response(
+ self,
+ model: str,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ResponsesAPIResponse:
+ """No transform applied since outputs are in OpenAI spec already"""
+ try:
+ logging_obj.post_call(
+ original_response=raw_response.text,
+ additional_args={"complete_input_dict": {}},
+ )
+ raw_response_json = raw_response.json()
+ raw_response_json["created_at"] = _safe_convert_created_field(
+ raw_response_json["created_at"]
+ )
+ except Exception:
+ raise OpenAIError(
+ message=raw_response.text, status_code=raw_response.status_code
+ )
+ raw_response_headers = dict(raw_response.headers)
+ processed_headers = process_response_headers(raw_response_headers)
+ try:
+ response = ResponsesAPIResponse(**raw_response_json)
+ except Exception:
+ verbose_logger.debug(
+ f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct"
+ )
+ response = ResponsesAPIResponse.model_construct(**raw_response_json)
+
+ # Store processed headers in additional_headers so they get returned to the client
+ response._hidden_params["additional_headers"] = processed_headers
+ response._hidden_params["headers"] = raw_response_headers
+ return response
+
+ def validate_environment(
+ self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
+ ) -> dict:
+ litellm_params = litellm_params or GenericLiteLLMParams()
+ api_key = (
+ litellm_params.api_key
+ or litellm.api_key
+ or litellm.openai_key
+ or get_secret_str("OPENAI_API_KEY")
+ )
+ headers.update(
+ {
+ "Authorization": f"Bearer {api_key}",
+ }
+ )
+ return headers
+
+ def get_complete_url(
+ self,
+ api_base: Optional[str],
+ litellm_params: dict,
+ ) -> str:
+ """
+ Get the endpoint for OpenAI responses API
+ """
+ api_base = (
+ api_base
+ or litellm.api_base
+ or get_secret_str("OPENAI_BASE_URL")
+ or get_secret_str("OPENAI_API_BASE")
+ or "https://api.openai.com/v1"
+ )
+
+ # Remove trailing slashes
+ api_base = api_base.rstrip("/")
+
+ return f"{api_base}/responses"
+
+ def transform_streaming_response(
+ self,
+ model: str,
+ parsed_chunk: dict,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ResponsesAPIStreamingResponse:
+ """
+ Transform a parsed streaming response chunk into a ResponsesAPIStreamingResponse
+ """
+ # Convert the dictionary to a properly typed ResponsesAPIStreamingResponse
+ verbose_logger.debug("Raw OpenAI Chunk=%s", parsed_chunk)
+ event_type = str(parsed_chunk.get("type"))
+ event_pydantic_model = OpenAIResponsesAPIConfig.get_event_model_class(
+ event_type=event_type
+ )
+ # Some OpenAI-compatible providers send error.code: null; coalesce so validation succeeds.
+ try:
+ error_obj = parsed_chunk.get("error")
+ if isinstance(error_obj, dict) and error_obj.get("code") is None:
+ parsed_chunk = dict(parsed_chunk)
+ parsed_chunk["error"] = dict(error_obj)
+ parsed_chunk["error"]["code"] = "unknown_error"
+ except Exception:
+ verbose_logger.debug("Failed to coalesce error.code in parsed_chunk")
+
+ try:
+ return event_pydantic_model(**parsed_chunk)
+ except ValidationError:
+ verbose_logger.debug(
+ "Pydantic validation failed for %s with chunk %s, "
+ "falling back to model_construct",
+ event_pydantic_model.__name__,
+ parsed_chunk,
+ )
+ return event_pydantic_model.model_construct(**parsed_chunk)
+
+ @staticmethod
+ def get_event_model_class(event_type: str) -> Any:
+ """
+ Returns the appropriate event model class based on the event type.
+
+ Args:
+ event_type (str): The type of event from the response chunk
+
+ Returns:
+ Any: The corresponding event model class
+
+ Raises:
+ ValueError: If the event type is unknown
+ """
+ event_models = {
+ ResponsesAPIStreamEvents.RESPONSE_CREATED: ResponseCreatedEvent,
+ ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS: ResponseInProgressEvent,
+ ResponsesAPIStreamEvents.RESPONSE_COMPLETED: ResponseCompletedEvent,
+ ResponsesAPIStreamEvents.RESPONSE_FAILED: ResponseFailedEvent,
+ ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE: ResponseIncompleteEvent,
+ ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED: OutputItemAddedEvent,
+ ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: OutputItemDoneEvent,
+ ResponsesAPIStreamEvents.CONTENT_PART_ADDED: ContentPartAddedEvent,
+ ResponsesAPIStreamEvents.CONTENT_PART_DONE: ContentPartDoneEvent,
+ ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA: OutputTextDeltaEvent,
+ ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED: OutputTextAnnotationAddedEvent,
+ ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE: OutputTextDoneEvent,
+ ResponsesAPIStreamEvents.REFUSAL_DELTA: RefusalDeltaEvent,
+ ResponsesAPIStreamEvents.REFUSAL_DONE: RefusalDoneEvent,
+ ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA: FunctionCallArgumentsDeltaEvent,
+ ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE: FunctionCallArgumentsDoneEvent,
+ ResponsesAPIStreamEvents.FILE_SEARCH_CALL_IN_PROGRESS: FileSearchCallInProgressEvent,
+ ResponsesAPIStreamEvents.FILE_SEARCH_CALL_SEARCHING: FileSearchCallSearchingEvent,
+ ResponsesAPIStreamEvents.FILE_SEARCH_CALL_COMPLETED: FileSearchCallCompletedEvent,
+ ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS: WebSearchCallInProgressEvent,
+ ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING: WebSearchCallSearchingEvent,
+ ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED: WebSearchCallCompletedEvent,
+ ResponsesAPIStreamEvents.MCP_LIST_TOOLS_IN_PROGRESS: MCPListToolsInProgressEvent,
+ ResponsesAPIStreamEvents.MCP_LIST_TOOLS_COMPLETED: MCPListToolsCompletedEvent,
+ ResponsesAPIStreamEvents.MCP_LIST_TOOLS_FAILED: MCPListToolsFailedEvent,
+ ResponsesAPIStreamEvents.MCP_CALL_IN_PROGRESS: MCPCallInProgressEvent,
+ ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA: MCPCallArgumentsDeltaEvent,
+ ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DONE: MCPCallArgumentsDoneEvent,
+ ResponsesAPIStreamEvents.MCP_CALL_COMPLETED: MCPCallCompletedEvent,
+ ResponsesAPIStreamEvents.MCP_CALL_FAILED: MCPCallFailedEvent,
+ ResponsesAPIStreamEvents.IMAGE_GENERATION_PARTIAL_IMAGE: ImageGenerationPartialImageEvent,
+ ResponsesAPIStreamEvents.ERROR: ErrorEvent,
+ # Shell tool events: passthrough as GenericEvent so payload is preserved
+ ResponsesAPIStreamEvents.SHELL_CALL_IN_PROGRESS: GenericEvent,
+ ResponsesAPIStreamEvents.SHELL_CALL_COMPLETED: GenericEvent,
+ ResponsesAPIStreamEvents.SHELL_CALL_OUTPUT: GenericEvent,
+ }
+
+ model_class = event_models.get(cast(ResponsesAPIStreamEvents, event_type))
+ if not model_class:
+ return GenericEvent
+
+ return model_class
+
+ def should_fake_stream(
+ self,
+ model: Optional[str],
+ stream: Optional[bool],
+ custom_llm_provider: Optional[str] = None,
+ ) -> bool:
+ if stream is not True:
+ return False
+ if model is not None:
+ try:
+ if (
+ litellm.utils.supports_native_streaming(
+ model=model,
+ custom_llm_provider=custom_llm_provider,
+ )
+ is False
+ ):
+ return True
+ except Exception as e:
+ verbose_logger.debug(
+ f"Error getting model info in OpenAIResponsesAPIConfig: {e}"
+ )
+ return False
+
+ def supports_native_websocket(self) -> bool:
+ """OpenAI supports native WebSocket for Responses API"""
+ return True
+
+ #########################################################
+ ########## DELETE RESPONSE API TRANSFORMATION ##############
+ #########################################################
+ def transform_delete_response_api_request(
+ self,
+ response_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """
+ Transform the delete response API request into a URL and data
+
+ OpenAI API expects the following request
+ - DELETE /v1/responses/{response_id}
+ """
+ url = f"{api_base}/{response_id}"
+ data: Dict = {}
+ return url, data
+
+ def transform_delete_response_api_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> DeleteResponseResult:
+ """
+ Transform the delete response API response into a DeleteResponseResult
+ """
+ try:
+ raw_response_json = raw_response.json()
+ except Exception:
+ raise OpenAIError(
+ message=raw_response.text, status_code=raw_response.status_code
+ )
+ return DeleteResponseResult(**raw_response_json)
+
+ #########################################################
+ ########## GET RESPONSE API TRANSFORMATION ###############
+ #########################################################
+ def transform_get_response_api_request(
+ self,
+ response_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """
+ Transform the get response API request into a URL and data
+
+ OpenAI API expects the following request
+ - GET /v1/responses/{response_id}
+ """
+ url = f"{api_base}/{response_id}"
+ data: Dict = {}
+ return url, data
+
+ def transform_get_response_api_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ResponsesAPIResponse:
+ """
+ Transform the get response API response into a ResponsesAPIResponse
+ """
+ try:
+ raw_response_json = raw_response.json()
+ except Exception:
+ raise OpenAIError(
+ message=raw_response.text, status_code=raw_response.status_code
+ )
+ raw_response_headers = dict(raw_response.headers)
+ processed_headers = process_response_headers(raw_response_headers)
+ response = ResponsesAPIResponse(**raw_response_json)
+ response._hidden_params["additional_headers"] = processed_headers
+ response._hidden_params["headers"] = raw_response_headers
+
+ return response
+
+ #########################################################
+ ########## LIST INPUT ITEMS TRANSFORMATION #############
+ #########################################################
+ def transform_list_input_items_request(
+ self,
+ response_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ after: Optional[str] = None,
+ before: Optional[str] = None,
+ include: Optional[List[str]] = None,
+ limit: int = 20,
+ order: Literal["asc", "desc"] = "desc",
+ ) -> Tuple[str, Dict]:
+ url = f"{api_base}/{response_id}/input_items"
+ params: Dict[str, Any] = {}
+ if after is not None:
+ params["after"] = after
+ if before is not None:
+ params["before"] = before
+ if include:
+ params["include"] = ",".join(include)
+ if limit is not None:
+ params["limit"] = limit
+ if order is not None:
+ params["order"] = order
+ return url, params
+
+ def transform_list_input_items_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> Dict:
+ try:
+ return raw_response.json()
+ except Exception:
+ raise OpenAIError(
+ message=raw_response.text, status_code=raw_response.status_code
+ )
+
+ #########################################################
+ ########## CANCEL RESPONSE API TRANSFORMATION ##########
+ #########################################################
+ def transform_cancel_response_api_request(
+ self,
+ response_id: str,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """
+ Transform the cancel response API request into a URL and data
+
+ OpenAI API expects the following request
+ - POST /v1/responses/{response_id}/cancel
+ """
+ url = f"{api_base}/{response_id}/cancel"
+ data: Dict = {}
+ return url, data
+
+ def transform_cancel_response_api_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ResponsesAPIResponse:
+ """
+ Transform the cancel response API response into a ResponsesAPIResponse
+ """
+ try:
+ raw_response_json = raw_response.json()
+ except Exception:
+ raise OpenAIError(
+ message=raw_response.text, status_code=raw_response.status_code
+ )
+ raw_response_headers = dict(raw_response.headers)
+ processed_headers = process_response_headers(raw_response_headers)
+
+ response = ResponsesAPIResponse(**raw_response_json)
+ response._hidden_params["additional_headers"] = processed_headers
+ response._hidden_params["headers"] = raw_response_headers
+
+ return response
+
+ #########################################################
+ ########## COMPACT RESPONSE API TRANSFORMATION ##########
+ #########################################################
+ def transform_compact_response_api_request(
+ self,
+ model: str,
+ input: Union[str, ResponseInputParam],
+ response_api_optional_request_params: Dict,
+ api_base: str,
+ litellm_params: GenericLiteLLMParams,
+ headers: dict,
+ ) -> Tuple[str, Dict]:
+ """
+ Transform the compact response API request into a URL and data
+
+ OpenAI API expects the following request
+ - POST /v1/responses/compact
+ """
+ # Preserve query params (e.g., api-version) while appending /compact.
+ parsed_url = httpx.URL(api_base)
+ compact_path = parsed_url.path.rstrip("/") + "/compact"
+ url = str(parsed_url.copy_with(path=compact_path))
+
+ input = self._validate_input_param(input)
+ data = dict(
+ ResponsesAPIRequestParams(
+ model=model, input=input, **response_api_optional_request_params
+ )
+ )
+
+ return url, data
+
+ def transform_compact_response_api_response(
+ self,
+ raw_response: httpx.Response,
+ logging_obj: LiteLLMLoggingObj,
+ ) -> ResponsesAPIResponse:
+ """
+ Transform the compact response API response into a ResponsesAPIResponse
+ """
+ try:
+ logging_obj.post_call(
+ original_response=raw_response.text,
+ additional_args={"complete_input_dict": {}},
+ )
+ raw_response_json = raw_response.json()
+ raw_response_json["created_at"] = _safe_convert_created_field(
+ raw_response_json["created_at"]
+ )
+ except Exception:
+ raise OpenAIError(
+ message=raw_response.text, status_code=raw_response.status_code
+ )
+ raw_response_headers = dict(raw_response.headers)
+ processed_headers = process_response_headers(raw_response_headers)
+
+ try:
+ response = ResponsesAPIResponse(**raw_response_json)
+ except Exception:
+ verbose_logger.debug(
+ f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct"
+ )
+ response = ResponsesAPIResponse.model_construct(**raw_response_json)
+
+ response._hidden_params["additional_headers"] = processed_headers
+ response._hidden_params["headers"] = raw_response_headers
+
+ return response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/speech/guardrail_translation/README.md b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/speech/guardrail_translation/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..52e89ffa9292d8484e4a9d469f88f6e3eca47158
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/speech/guardrail_translation/README.md
@@ -0,0 +1,178 @@
+# OpenAI Text-to-Speech Guardrail Translation Handler
+
+Handler for processing OpenAI's text-to-speech endpoint (`/v1/audio/speech`) with guardrails.
+
+## Overview
+
+This handler processes text-to-speech requests by:
+1. Extracting the input text from the request
+2. Applying guardrails to the input text
+3. Updating the request with the guardrailed text
+4. Returning the output unchanged (audio is binary, not text)
+
+## Data Format
+
+### Input Format
+
+```json
+{
+ "model": "tts-1",
+ "input": "The quick brown fox jumped over the lazy dog.",
+ "voice": "alloy",
+ "response_format": "mp3",
+ "speed": 1.0
+}
+```
+
+### Output Format
+
+The output is binary audio data (MP3, WAV, etc.), not text, so it cannot be guardrailed.
+
+## Usage
+
+The handler is automatically discovered and applied when guardrails are used with the text-to-speech endpoint.
+
+### Example: Using Guardrails with Text-to-Speech
+
+```bash
+curl -X POST 'http://localhost:4000/v1/audio/speech' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer your-api-key' \
+-d '{
+ "model": "tts-1",
+ "input": "The quick brown fox jumped over the lazy dog.",
+ "voice": "alloy",
+ "guardrails": ["content_moderation"]
+}' \
+--output speech.mp3
+```
+
+The guardrail will be applied to the input text before the text-to-speech conversion.
+
+### Example: PII Masking in TTS Input
+
+```bash
+curl -X POST 'http://localhost:4000/v1/audio/speech' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer your-api-key' \
+-d '{
+ "model": "tts-1",
+ "input": "Please call John Doe at john@example.com",
+ "voice": "nova",
+ "guardrails": ["mask_pii"]
+}' \
+--output speech.mp3
+```
+
+The audio will say: "Please call [NAME_REDACTED] at [EMAIL_REDACTED]"
+
+### Example: Content Filtering Before TTS
+
+```bash
+curl -X POST 'http://localhost:4000/v1/audio/speech' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer your-api-key' \
+-d '{
+ "model": "tts-1-hd",
+ "input": "This is the text that will be spoken",
+ "voice": "shimmer",
+ "guardrails": ["content_filter"]
+}' \
+--output speech.mp3
+```
+
+## Implementation Details
+
+### Input Processing
+
+- **Field**: `input` (string)
+- **Processing**: Applies guardrail to input text
+- **Result**: Updated input text in request
+
+### Output Processing
+
+- **Processing**: Not applicable (audio is binary data)
+- **Result**: Response returned unchanged
+
+## Use Cases
+
+1. **PII Protection**: Remove personally identifiable information before converting to speech
+2. **Content Filtering**: Remove inappropriate content before TTS conversion
+3. **Compliance**: Ensure text meets requirements before voice synthesis
+4. **Text Sanitization**: Clean up text before audio generation
+
+## Extension
+
+Override these methods to customize behavior:
+
+- `process_input_messages()`: Customize how input text is processed
+- `process_output_response()`: Currently a no-op, but can be overridden if needed
+
+## Supported Call Types
+
+- `CallTypes.speech` - Synchronous text-to-speech
+- `CallTypes.aspeech` - Asynchronous text-to-speech
+
+## Notes
+
+- Only the input text is processed by guardrails
+- Output processing is a no-op since audio cannot be text-guardrailed
+- Both sync and async call types use the same handler
+- Works with all TTS models (tts-1, tts-1-hd, etc.)
+- Works with all voice options
+
+## Common Patterns
+
+### Remove PII Before TTS
+
+```python
+import litellm
+from pathlib import Path
+
+speech_file_path = Path(__file__).parent / "speech.mp3"
+response = litellm.speech(
+ model="tts-1",
+ voice="alloy",
+ input="Hi, this is John Doe calling from john@company.com",
+ guardrails=["mask_pii"],
+)
+response.stream_to_file(speech_file_path)
+# Audio will have PII masked
+```
+
+### Content Moderation Before TTS
+
+```python
+import litellm
+from pathlib import Path
+
+speech_file_path = Path(__file__).parent / "speech.mp3"
+response = litellm.speech(
+ model="tts-1-hd",
+ voice="nova",
+ input="Your text here",
+ guardrails=["content_moderation"],
+)
+response.stream_to_file(speech_file_path)
+```
+
+### Async TTS with Guardrails
+
+```python
+import litellm
+import asyncio
+from pathlib import Path
+
+async def generate_speech():
+ speech_file_path = Path(__file__).parent / "speech.mp3"
+ response = await litellm.aspeech(
+ model="tts-1",
+ voice="echo",
+ input="Text to convert to speech",
+ guardrails=["pii_mask"],
+ )
+ response.stream_to_file(speech_file_path)
+
+asyncio.run(generate_speech())
+```
+
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/speech/guardrail_translation/__init__.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/speech/guardrail_translation/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..ef7d50f861af6438d9499adb242aec6e3fb1786f
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/speech/guardrail_translation/__init__.py
@@ -0,0 +1,13 @@
+"""OpenAI Text-to-Speech handler for Unified Guardrails."""
+
+from litellm.llms.openai.speech.guardrail_translation.handler import (
+ OpenAITextToSpeechHandler,
+)
+from litellm.types.utils import CallTypes
+
+guardrail_translation_mappings = {
+ CallTypes.speech: OpenAITextToSpeechHandler,
+ CallTypes.aspeech: OpenAITextToSpeechHandler,
+}
+
+__all__ = ["guardrail_translation_mappings", "OpenAITextToSpeechHandler"]
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/speech/guardrail_translation/handler.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/speech/guardrail_translation/handler.py
new file mode 100644
index 0000000000000000000000000000000000000000..e6796fbac2a4e4229b6db8073508cafe9e823854
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/speech/guardrail_translation/handler.py
@@ -0,0 +1,108 @@
+"""
+OpenAI Text-to-Speech Handler for Unified Guardrails
+
+This module provides guardrail translation support for OpenAI's text-to-speech endpoint.
+The handler processes the 'input' text parameter (output is audio, so no text to guardrail).
+"""
+
+from typing import TYPE_CHECKING, Any, Optional
+
+from litellm._logging import verbose_proxy_logger
+from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
+from litellm.types.utils import GenericGuardrailAPIInputs
+
+if TYPE_CHECKING:
+ from litellm.integrations.custom_guardrail import CustomGuardrail
+ from litellm.types.llms.openai import HttpxBinaryResponseContent
+
+
+class OpenAITextToSpeechHandler(BaseTranslation):
+ """
+ Handler for processing OpenAI text-to-speech requests with guardrails.
+
+ This class provides methods to:
+ 1. Process input text (pre-call hook)
+
+ Note: Output processing is not applicable since the output is audio (binary),
+ not text. Only the input text is processed.
+ """
+
+ async def process_input_messages(
+ self,
+ data: dict,
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process input text by applying guardrails.
+
+ Args:
+ data: Request data dictionary containing 'input' parameter
+ guardrail_to_apply: The guardrail instance to apply
+
+ Returns:
+ Modified data with guardrails applied to input text
+ """
+ input_text = data.get("input")
+ if input_text is None:
+ verbose_proxy_logger.debug(
+ "OpenAI Text-to-Speech: No input text found in request data"
+ )
+ return data
+
+ if isinstance(input_text, str):
+ inputs = GenericGuardrailAPIInputs(texts=[input_text])
+ # Include model information if available (voice model)
+ model = data.get("model")
+ if model:
+ inputs["model"] = model
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=data,
+ input_type="request",
+ logging_obj=litellm_logging_obj,
+ )
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+ data["input"] = guardrailed_texts[0] if guardrailed_texts else input_text
+
+ verbose_proxy_logger.debug(
+ "OpenAI Text-to-Speech: Applied guardrail to input text. "
+ "Original length: %d, New length: %d",
+ len(input_text),
+ len(data["input"]),
+ )
+ else:
+ verbose_proxy_logger.debug(
+ "OpenAI Text-to-Speech: Unexpected input type: %s. Expected string.",
+ type(input_text),
+ )
+
+ return data
+
+ async def process_output_response(
+ self,
+ response: "HttpxBinaryResponseContent",
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional[Any] = None,
+ user_api_key_dict: Optional[Any] = None,
+ ) -> Any:
+ """
+ Process output - not applicable for text-to-speech.
+
+ The output is audio (binary data), not text, so there's nothing to apply
+ guardrails to. This method returns the response unchanged.
+
+ Args:
+ response: Binary audio response
+ guardrail_to_apply: The guardrail instance (unused)
+ litellm_logging_obj: Optional logging object (unused)
+ user_api_key_dict: User API key metadata (unused)
+
+ Returns:
+ Unmodified response (audio data doesn't need text guardrails)
+ """
+ verbose_proxy_logger.debug(
+ "OpenAI Text-to-Speech: Output processing not applicable "
+ "(output is audio data, not text)"
+ )
+ return response
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/transcriptions/gpt_transformation.py b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/transcriptions/gpt_transformation.py
new file mode 100644
index 0000000000000000000000000000000000000000..34621c44e22d61857d02590bc3bdaf3da0253305
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/transcriptions/gpt_transformation.py
@@ -0,0 +1,41 @@
+from typing import List
+
+from litellm.llms.base_llm.audio_transcription.transformation import (
+ AudioTranscriptionRequestData,
+)
+from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
+from litellm.types.utils import FileTypes
+
+from .whisper_transformation import OpenAIWhisperAudioTranscriptionConfig
+
+
+class OpenAIGPTAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
+ def get_supported_openai_params(
+ self, model: str
+ ) -> List[OpenAIAudioTranscriptionOptionalParams]:
+ """
+ Get the supported OpenAI params for the `gpt-4o-transcribe` models
+ """
+ return [
+ "language",
+ "prompt",
+ "response_format",
+ "temperature",
+ "include",
+ ]
+
+ def transform_audio_transcription_request(
+ self,
+ model: str,
+ audio_file: FileTypes,
+ optional_params: dict,
+ litellm_params: dict,
+ ) -> AudioTranscriptionRequestData:
+ """
+ Transform the audio transcription request
+ """
+ data = {"model": model, "file": audio_file, **optional_params}
+
+ return AudioTranscriptionRequestData(
+ data=data,
+ )
diff --git a/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/transcriptions/guardrail_translation/README.md b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/transcriptions/guardrail_translation/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..08e5b6f85c50d094c97d634a97ceae85868d0fc2
--- /dev/null
+++ b/loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/openai/transcriptions/guardrail_translation/README.md
@@ -0,0 +1,159 @@
+# OpenAI Audio Transcription Guardrail Translation Handler
+
+Handler for processing OpenAI's audio transcription endpoint (`/v1/audio/transcriptions`) with guardrails.
+
+## Overview
+
+This handler processes audio transcription responses by:
+1. Applying guardrails to the transcribed text output
+2. Returning the input unchanged (since input is an audio file, not text)
+
+## Data Format
+
+### Input Format
+
+The input is an audio file, which cannot be guardrailed (it's binary data, not text).
+
+```json
+{
+ "model": "whisper-1",
+ "file": "