Add loan-advisor-agent (part 4)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/README.md +12 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/chat/transformation.py +148 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/image_edit/transformation.py +26 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/image_generation/transformation.py +41 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/responses/transformation.py +52 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/README.md +381 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/__init__.py +54 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/code_execution.py +317 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/constants.py +12 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/handler.py +219 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/prompt_injection.py +308 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/sandbox_executor.py +286 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/transformation.py +345 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/llamafile/chat/transformation.py +45 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/lm_studio/chat/transformation.py +49 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/lm_studio/embed/transformation.py +50 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/__init__.py +1 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/files/__init__.py +1 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/files/transformation.py +434 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/responses/__init__.py +1 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/responses/transformation.py +353 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/maritalk.py +65 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/meta_llama/chat/transformation.py +46 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/milvus/vector_stores/__init__.py +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/milvus/vector_stores/transformation.py +281 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/__init__.py +13 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/chat/__init__.py +3 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/chat/transformation.py +101 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/messages/transformation.py +76 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/text_to_speech/__init__.py +7 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/text_to_speech/transformation.py +414 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/audio_transcription/transformation.py +159 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/chat/transformation.py +686 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/embedding.py +4 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/mistral_embedding_transformation.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/__init__.py +1 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/guardrail_translation/__init__.py +11 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/guardrail_translation/handler.py +153 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/transformation.py +225 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/moonshot/chat/transformation.py +246 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/__init__.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/chat/__init__.py +0 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/chat/transformation.py +40 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nebius/chat/transformation.py +27 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nebius/embedding/transformation.py +5 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/chat/handler.py +131 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/chat/transformation.py +228 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/common_utils.py +15 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/novita/chat/transformation.py +33 -0
- loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nscale/chat/transformation.py +52 -0
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/README.md
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## File Structure
|
| 2 |
+
|
| 3 |
+
### August 27th, 2024
|
| 4 |
+
|
| 5 |
+
To make it easy to see how calls are transformed for each model/provider:
|
| 6 |
+
|
| 7 |
+
we are working on moving all supported litellm providers to a folder structure, where folder name is the supported litellm provider name.
|
| 8 |
+
|
| 9 |
+
Each folder will contain a `*_transformation.py` file, which has all the request/response transformation logic, making it easy to see how calls are modified.
|
| 10 |
+
|
| 11 |
+
E.g. `cohere/`, `bedrock/`.
|
| 12 |
+
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/chat/transformation.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from typing import TYPE_CHECKING, List, Optional, Tuple
|
| 6 |
+
|
| 7 |
+
from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
|
| 8 |
+
from litellm.secret_managers.main import get_secret_bool, get_secret_str
|
| 9 |
+
from litellm.types.router import LiteLLM_Params
|
| 10 |
+
|
| 11 |
+
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 12 |
+
|
| 13 |
+
if TYPE_CHECKING:
|
| 14 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class LiteLLMProxyChatConfig(OpenAIGPTConfig):
|
| 18 |
+
def get_supported_openai_params(self, model: str) -> List:
|
| 19 |
+
params_list = super().get_supported_openai_params(model)
|
| 20 |
+
params_list.extend(OPENAI_CHAT_COMPLETION_PARAMS)
|
| 21 |
+
return params_list
|
| 22 |
+
|
| 23 |
+
def _map_openai_params(
|
| 24 |
+
self,
|
| 25 |
+
non_default_params: dict,
|
| 26 |
+
optional_params: dict,
|
| 27 |
+
model: str,
|
| 28 |
+
drop_params: bool,
|
| 29 |
+
) -> dict:
|
| 30 |
+
supported_openai_params = self.get_supported_openai_params(model)
|
| 31 |
+
for param, value in non_default_params.items():
|
| 32 |
+
if param == "thinking":
|
| 33 |
+
optional_params.setdefault("extra_body", {})["thinking"] = value
|
| 34 |
+
elif param in supported_openai_params:
|
| 35 |
+
optional_params[param] = value
|
| 36 |
+
return optional_params
|
| 37 |
+
|
| 38 |
+
def _get_openai_compatible_provider_info(
|
| 39 |
+
self, api_base: Optional[str], api_key: Optional[str]
|
| 40 |
+
) -> Tuple[Optional[str], Optional[str]]:
|
| 41 |
+
api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE") # type: ignore
|
| 42 |
+
dynamic_api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY")
|
| 43 |
+
return api_base, dynamic_api_key
|
| 44 |
+
|
| 45 |
+
def get_models(
|
| 46 |
+
self, api_key: Optional[str] = None, api_base: Optional[str] = None
|
| 47 |
+
) -> List[str]:
|
| 48 |
+
api_base, api_key = self._get_openai_compatible_provider_info(api_base, api_key)
|
| 49 |
+
if api_base is None:
|
| 50 |
+
raise ValueError(
|
| 51 |
+
"api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`"
|
| 52 |
+
)
|
| 53 |
+
models = super().get_models(api_key=api_key, api_base=api_base)
|
| 54 |
+
return [f"litellm_proxy/{model}" for model in models]
|
| 55 |
+
|
| 56 |
+
@staticmethod
|
| 57 |
+
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
|
| 58 |
+
return api_key or get_secret_str("LITELLM_PROXY_API_KEY")
|
| 59 |
+
|
| 60 |
+
@staticmethod
|
| 61 |
+
def _should_use_litellm_proxy_by_default(
|
| 62 |
+
litellm_params: Optional[LiteLLM_Params] = None,
|
| 63 |
+
):
|
| 64 |
+
"""
|
| 65 |
+
Returns True if litellm proxy should be used by default for a given request
|
| 66 |
+
|
| 67 |
+
Issue: https://github.com/BerriAI/litellm/issues/10559
|
| 68 |
+
|
| 69 |
+
Use case:
|
| 70 |
+
- When using Google ADK, users want a flag to dynamically enable sending the request to litellm proxy or not
|
| 71 |
+
- Allow the model name to be passed in original format and still use litellm proxy:
|
| 72 |
+
"gemini/gemini-1.5-pro", "openai/gpt-4", "mistral/llama-2-70b-chat" etc.
|
| 73 |
+
"""
|
| 74 |
+
import litellm
|
| 75 |
+
|
| 76 |
+
if get_secret_bool("USE_LITELLM_PROXY") is True:
|
| 77 |
+
return True
|
| 78 |
+
if litellm_params and litellm_params.use_litellm_proxy is True:
|
| 79 |
+
return True
|
| 80 |
+
if litellm.use_litellm_proxy is True:
|
| 81 |
+
return True
|
| 82 |
+
return False
|
| 83 |
+
|
| 84 |
+
@staticmethod
|
| 85 |
+
def litellm_proxy_get_custom_llm_provider_info(
|
| 86 |
+
model: str, api_base: Optional[str] = None, api_key: Optional[str] = None
|
| 87 |
+
) -> Tuple[str, str, Optional[str], Optional[str]]:
|
| 88 |
+
"""
|
| 89 |
+
Force use litellm proxy for all models
|
| 90 |
+
|
| 91 |
+
Issue: https://github.com/BerriAI/litellm/issues/10559
|
| 92 |
+
|
| 93 |
+
Expected behavior:
|
| 94 |
+
- custom_llm_provider will be 'litellm_proxy'
|
| 95 |
+
- api_base = api_base OR LITELLM_PROXY_API_BASE
|
| 96 |
+
- api_key = api_key OR LITELLM_PROXY_API_KEY
|
| 97 |
+
|
| 98 |
+
Use case:
|
| 99 |
+
- When using Google ADK, users want a flag to dynamically enable sending the request to litellm proxy or not
|
| 100 |
+
- Allow the model name to be passed in original format and still use litellm proxy:
|
| 101 |
+
"gemini/gemini-1.5-pro", "openai/gpt-4", "mistral/llama-2-70b-chat" etc.
|
| 102 |
+
|
| 103 |
+
Return model, custom_llm_provider, dynamic_api_key, api_base
|
| 104 |
+
"""
|
| 105 |
+
import litellm
|
| 106 |
+
|
| 107 |
+
custom_llm_provider = "litellm_proxy"
|
| 108 |
+
if model.startswith("litellm_proxy/"):
|
| 109 |
+
model = model.split("/", 1)[1]
|
| 110 |
+
|
| 111 |
+
(
|
| 112 |
+
api_base,
|
| 113 |
+
api_key,
|
| 114 |
+
) = litellm.LiteLLMProxyChatConfig()._get_openai_compatible_provider_info(
|
| 115 |
+
api_base=api_base, api_key=api_key
|
| 116 |
+
)
|
| 117 |
+
|
| 118 |
+
return model, custom_llm_provider, api_key, api_base
|
| 119 |
+
|
| 120 |
+
def transform_request(
|
| 121 |
+
self,
|
| 122 |
+
model: str,
|
| 123 |
+
messages: List["AllMessageValues"],
|
| 124 |
+
optional_params: dict,
|
| 125 |
+
litellm_params: dict,
|
| 126 |
+
headers: dict,
|
| 127 |
+
) -> dict:
|
| 128 |
+
# don't transform the request
|
| 129 |
+
return {
|
| 130 |
+
"model": model,
|
| 131 |
+
"messages": messages,
|
| 132 |
+
**optional_params,
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
async def async_transform_request(
|
| 136 |
+
self,
|
| 137 |
+
model: str,
|
| 138 |
+
messages: List["AllMessageValues"],
|
| 139 |
+
optional_params: dict,
|
| 140 |
+
litellm_params: dict,
|
| 141 |
+
headers: dict,
|
| 142 |
+
) -> dict:
|
| 143 |
+
# don't transform the request
|
| 144 |
+
return {
|
| 145 |
+
"model": model,
|
| 146 |
+
"messages": messages,
|
| 147 |
+
**optional_params,
|
| 148 |
+
}
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/image_edit/transformation.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional
|
| 2 |
+
|
| 3 |
+
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
|
| 4 |
+
from litellm.secret_managers.main import get_secret_str
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class LiteLLMProxyImageEditConfig(OpenAIImageEditConfig):
|
| 8 |
+
"""Configuration for image edit requests routed through LiteLLM Proxy."""
|
| 9 |
+
|
| 10 |
+
def validate_environment(
|
| 11 |
+
self, headers: dict, model: str, api_key: Optional[str] = None
|
| 12 |
+
) -> dict:
|
| 13 |
+
api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY")
|
| 14 |
+
headers.update({"Authorization": f"Bearer {api_key}"})
|
| 15 |
+
return headers
|
| 16 |
+
|
| 17 |
+
def get_complete_url(
|
| 18 |
+
self, model: str, api_base: Optional[str], litellm_params: dict
|
| 19 |
+
) -> str:
|
| 20 |
+
api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE")
|
| 21 |
+
if api_base is None:
|
| 22 |
+
raise ValueError(
|
| 23 |
+
"api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`"
|
| 24 |
+
)
|
| 25 |
+
api_base = api_base.rstrip("/")
|
| 26 |
+
return f"{api_base}/images/edits"
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/image_generation/transformation.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional
|
| 2 |
+
|
| 3 |
+
from litellm.llms.openai.image_generation.gpt_transformation import (
|
| 4 |
+
GPTImageGenerationConfig,
|
| 5 |
+
)
|
| 6 |
+
from litellm.secret_managers.main import get_secret_str
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class LiteLLMProxyImageGenerationConfig(GPTImageGenerationConfig):
|
| 10 |
+
"""Configuration for image generation requests routed through LiteLLM Proxy."""
|
| 11 |
+
|
| 12 |
+
def validate_environment(
|
| 13 |
+
self,
|
| 14 |
+
headers: dict,
|
| 15 |
+
model: str,
|
| 16 |
+
messages,
|
| 17 |
+
optional_params: dict,
|
| 18 |
+
litellm_params: dict,
|
| 19 |
+
api_key: Optional[str] = None,
|
| 20 |
+
api_base: Optional[str] = None,
|
| 21 |
+
) -> dict:
|
| 22 |
+
api_key = api_key or get_secret_str("LITELLM_PROXY_API_KEY")
|
| 23 |
+
headers.update({"Authorization": f"Bearer {api_key}"})
|
| 24 |
+
return headers
|
| 25 |
+
|
| 26 |
+
def get_complete_url(
|
| 27 |
+
self,
|
| 28 |
+
api_base: Optional[str],
|
| 29 |
+
api_key: Optional[str],
|
| 30 |
+
model: str,
|
| 31 |
+
optional_params: dict,
|
| 32 |
+
litellm_params: dict,
|
| 33 |
+
stream: Optional[bool] = None,
|
| 34 |
+
) -> str:
|
| 35 |
+
api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE")
|
| 36 |
+
if api_base is None:
|
| 37 |
+
raise ValueError(
|
| 38 |
+
"api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`"
|
| 39 |
+
)
|
| 40 |
+
api_base = api_base.rstrip("/")
|
| 41 |
+
return f"{api_base}/images/generations"
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/responses/transformation.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Responses API transformation for LiteLLM Proxy provider.
|
| 3 |
+
|
| 4 |
+
LiteLLM Proxy supports the OpenAI Responses API natively when the underlying model supports it.
|
| 5 |
+
This config enables pass-through behavior to the proxy's /v1/responses endpoint.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import Optional
|
| 9 |
+
|
| 10 |
+
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
| 11 |
+
from litellm.secret_managers.main import get_secret_str
|
| 12 |
+
from litellm.types.utils import LlmProviders
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
| 16 |
+
"""
|
| 17 |
+
Configuration for LiteLLM Proxy Responses API support.
|
| 18 |
+
|
| 19 |
+
Extends OpenAI's config since the proxy follows OpenAI's API spec,
|
| 20 |
+
but uses LITELLM_PROXY_API_BASE for the base URL.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
@property
|
| 24 |
+
def custom_llm_provider(self) -> LlmProviders:
|
| 25 |
+
return LlmProviders.LITELLM_PROXY
|
| 26 |
+
|
| 27 |
+
def get_complete_url(
|
| 28 |
+
self,
|
| 29 |
+
api_base: Optional[str],
|
| 30 |
+
litellm_params: dict,
|
| 31 |
+
) -> str:
|
| 32 |
+
"""
|
| 33 |
+
Get the endpoint for LiteLLM Proxy responses API.
|
| 34 |
+
|
| 35 |
+
Uses LITELLM_PROXY_API_BASE environment variable if api_base is not provided.
|
| 36 |
+
"""
|
| 37 |
+
api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE")
|
| 38 |
+
|
| 39 |
+
if api_base is None:
|
| 40 |
+
raise ValueError(
|
| 41 |
+
"api_base not set for LiteLLM Proxy responses API. "
|
| 42 |
+
"Set via api_base parameter or LITELLM_PROXY_API_BASE environment variable"
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
# Remove trailing slashes
|
| 46 |
+
api_base = api_base.rstrip("/")
|
| 47 |
+
|
| 48 |
+
return f"{api_base}/responses"
|
| 49 |
+
|
| 50 |
+
def supports_native_websocket(self) -> bool:
|
| 51 |
+
"""LiteLLM Proxy does not support native WebSocket for Responses API"""
|
| 52 |
+
return False
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/README.md
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# LiteLLM Skills - Database-Backed Skills Storage
|
| 2 |
+
|
| 3 |
+
This module provides database-backed skills storage as an alternative to Anthropic's cloud-based Skills API. It enables using skills with **any LLM provider** (Bedrock, OpenAI, Azure, etc.) by storing skills locally and converting them to tools + system prompt injection.
|
| 4 |
+
|
| 5 |
+
## Architecture
|
| 6 |
+
|
| 7 |
+
```mermaid
|
| 8 |
+
flowchart TB
|
| 9 |
+
subgraph "Skill Creation"
|
| 10 |
+
A[User creates skill with ZIP file] --> B{custom_llm_provider?}
|
| 11 |
+
B -->|anthropic| C[Forward to Anthropic API]
|
| 12 |
+
B -->|litellm_proxy| D[Store in LiteLLM Database]
|
| 13 |
+
|
| 14 |
+
D --> E[Extract & store:<br/>- display_title<br/>- description<br/>- instructions<br/>- file_content ZIP]
|
| 15 |
+
end
|
| 16 |
+
|
| 17 |
+
subgraph "Skill Usage in Messages API"
|
| 18 |
+
F[Request with container.skills] --> G[SkillsInjectionHook]
|
| 19 |
+
G --> H{skill_id prefix?}
|
| 20 |
+
|
| 21 |
+
H -->|"litellm:skill_abc"| I[Fetch from LiteLLM DB]
|
| 22 |
+
H -->|"skill_xyz" no prefix| J[Pass to Anthropic as native skill]
|
| 23 |
+
|
| 24 |
+
I --> K{Model provider?}
|
| 25 |
+
K -->|Anthropic API| L[Convert to tools]
|
| 26 |
+
K -->|Bedrock/OpenAI/etc| M[Convert to tools +<br/>Inject SKILL.md into system prompt]
|
| 27 |
+
|
| 28 |
+
J --> N[Keep in container.skills]
|
| 29 |
+
end
|
| 30 |
+
|
| 31 |
+
subgraph "Skill Resolution for Non-Anthropic"
|
| 32 |
+
M --> O[Extract SKILL.md from ZIP]
|
| 33 |
+
O --> P[Add to system prompt:<br/># Available Skills<br/>## Skill: My Skill<br/>SKILL.md content...]
|
| 34 |
+
P --> Q[Create OpenAI-style tool:<br/>type: function<br/>name: skill_id<br/>description: instructions]
|
| 35 |
+
Q --> R[Send to LLM Provider]
|
| 36 |
+
end
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
## Automatic Code Execution
|
| 40 |
+
|
| 41 |
+
For skills that include executable code (Python files), LiteLLM automatically handles:
|
| 42 |
+
|
| 43 |
+
1. **Pre-call hook** (`async_pre_call_hook`): Adds `litellm_code_execution` tool, injects SKILL.md content
|
| 44 |
+
2. **Post-call hook** (`async_post_call_success_deployment_hook`): Detects tool calls, executes code in Docker sandbox, continues loop
|
| 45 |
+
3. **Returns files**: Generated files (GIFs, images, etc.) returned directly on response
|
| 46 |
+
|
| 47 |
+
```mermaid
|
| 48 |
+
sequenceDiagram
|
| 49 |
+
participant User
|
| 50 |
+
participant LiteLLM as LiteLLM SDK
|
| 51 |
+
participant PreHook as async_pre_call_hook
|
| 52 |
+
participant LLM as LLM Provider
|
| 53 |
+
participant PostHook as async_post_call_success_deployment_hook
|
| 54 |
+
participant Sandbox as Docker Sandbox
|
| 55 |
+
|
| 56 |
+
User->>LiteLLM: litellm.acompletion(model, messages, container={skills: [...]})
|
| 57 |
+
|
| 58 |
+
Note over LiteLLM,PreHook: PRE-CALL HOOK
|
| 59 |
+
LiteLLM->>PreHook: Intercept request
|
| 60 |
+
PreHook->>PreHook: Fetch skill from DB (litellm:skill_id)
|
| 61 |
+
PreHook->>PreHook: Extract SKILL.md from ZIP
|
| 62 |
+
PreHook->>PreHook: Inject SKILL.md into system prompt
|
| 63 |
+
PreHook->>PreHook: Add litellm_code_execution tool
|
| 64 |
+
PreHook->>PreHook: Store skill files in metadata
|
| 65 |
+
PreHook-->>LiteLLM: Modified request
|
| 66 |
+
|
| 67 |
+
LiteLLM->>LLM: Forward to provider (OpenAI/Bedrock/etc)
|
| 68 |
+
LLM-->>LiteLLM: Response with tool_calls
|
| 69 |
+
|
| 70 |
+
Note over LiteLLM,PostHook: POST-CALL HOOK (Agentic Loop)
|
| 71 |
+
LiteLLM->>PostHook: Check response
|
| 72 |
+
|
| 73 |
+
loop Until no more tool calls
|
| 74 |
+
PostHook->>PostHook: Check for litellm_code_execution tool call
|
| 75 |
+
alt Has code execution tool call
|
| 76 |
+
PostHook->>Sandbox: Execute Python code
|
| 77 |
+
Sandbox->>Sandbox: Copy skill files to /sandbox
|
| 78 |
+
Sandbox->>Sandbox: Install requirements.txt
|
| 79 |
+
Sandbox->>Sandbox: Run code
|
| 80 |
+
Sandbox-->>PostHook: Result + generated files
|
| 81 |
+
PostHook->>PostHook: Add tool result to messages
|
| 82 |
+
PostHook->>LLM: Make another LLM call
|
| 83 |
+
LLM-->>PostHook: New response
|
| 84 |
+
else No code execution
|
| 85 |
+
PostHook->>PostHook: Break loop
|
| 86 |
+
end
|
| 87 |
+
end
|
| 88 |
+
|
| 89 |
+
PostHook->>PostHook: Attach files to response._litellm_generated_files
|
| 90 |
+
PostHook-->>LiteLLM: Modified response with files
|
| 91 |
+
LiteLLM-->>User: Final response with generated files
|
| 92 |
+
```
|
| 93 |
+
|
| 94 |
+
```python
|
| 95 |
+
import litellm
|
| 96 |
+
from litellm.proxy.hooks.litellm_skills import SkillsInjectionHook
|
| 97 |
+
|
| 98 |
+
# Register the hook (done once at startup)
|
| 99 |
+
hook = SkillsInjectionHook()
|
| 100 |
+
litellm.callbacks.append(hook)
|
| 101 |
+
|
| 102 |
+
# ONE request - LiteLLM handles everything automatically
|
| 103 |
+
# The container parameter triggers the SkillsInjectionHook
|
| 104 |
+
response = await litellm.acompletion(
|
| 105 |
+
model="gpt-4o-mini",
|
| 106 |
+
messages=[{"role": "user", "content": "Create a bouncing ball GIF"}],
|
| 107 |
+
container={
|
| 108 |
+
"skills": [{"type": "custom", "skill_id": "litellm:skill_abc123"}]
|
| 109 |
+
},
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
# Files are attached directly to response
|
| 113 |
+
generated_files = response._litellm_generated_files
|
| 114 |
+
for f in generated_files:
|
| 115 |
+
print(f"Generated: {f['name']} ({f['size']} bytes)")
|
| 116 |
+
# f['content_base64'] contains the file data
|
| 117 |
+
```
|
| 118 |
+
|
| 119 |
+
This mimics Anthropic's behavior - no manual agentic loop needed!
|
| 120 |
+
|
| 121 |
+
### How it works
|
| 122 |
+
|
| 123 |
+
The `SkillsInjectionHook` uses two hooks:
|
| 124 |
+
|
| 125 |
+
1. **`async_pre_call_hook`** (proxy only): Transforms the request before LLM call
|
| 126 |
+
- Fetches skills from DB
|
| 127 |
+
- Injects SKILL.md into system prompt
|
| 128 |
+
- Adds `litellm_code_execution` tool
|
| 129 |
+
- Sets `_litellm_code_execution_enabled=True` in metadata
|
| 130 |
+
|
| 131 |
+
2. **`async_post_call_success_deployment_hook`** (SDK + proxy): Called after LLM response
|
| 132 |
+
- Checks if response has `litellm_code_execution` tool call
|
| 133 |
+
- Executes code in Docker sandbox
|
| 134 |
+
- Adds result to messages, makes another LLM call
|
| 135 |
+
- Repeats until model gives final response
|
| 136 |
+
- Attaches generated files to `response._litellm_generated_files`
|
| 137 |
+
|
| 138 |
+
## File Structure
|
| 139 |
+
|
| 140 |
+
```
|
| 141 |
+
litellm/llms/litellm_proxy/skills/
|
| 142 |
+
├── __init__.py # Exports all skill components
|
| 143 |
+
├── handler.py # LiteLLMSkillsHandler - database CRUD operations (Prisma)
|
| 144 |
+
├── transformation.py # LiteLLMSkillsTransformationHandler - SDK transformation layer
|
| 145 |
+
├── prompt_injection.py # SkillPromptInjectionHandler - SKILL.md extraction and injection
|
| 146 |
+
├── sandbox_executor.py # SkillsSandboxExecutor - Docker sandbox code execution
|
| 147 |
+
├── code_execution.py # CodeExecutionHandler - automatic agentic loop
|
| 148 |
+
└── README.md # This file
|
| 149 |
+
|
| 150 |
+
litellm/proxy/hooks/litellm_skills/
|
| 151 |
+
├── __init__.py # Re-exports from SDK + SkillsInjectionHook
|
| 152 |
+
└── main.py # SkillsInjectionHook - CustomLogger hook for proxy
|
| 153 |
+
```
|
| 154 |
+
|
| 155 |
+
## Components
|
| 156 |
+
|
| 157 |
+
### 1. `handler.py` - LiteLLMSkillsHandler
|
| 158 |
+
|
| 159 |
+
Database operations for skills CRUD:
|
| 160 |
+
|
| 161 |
+
```python
|
| 162 |
+
from litellm.llms.litellm_proxy.skills import LiteLLMSkillsHandler
|
| 163 |
+
|
| 164 |
+
# Create skill
|
| 165 |
+
skill = await LiteLLMSkillsHandler.create_skill(
|
| 166 |
+
data=NewSkillRequest(
|
| 167 |
+
display_title="My Skill",
|
| 168 |
+
description="A helpful skill",
|
| 169 |
+
instructions="Use this skill when...",
|
| 170 |
+
file_content=zip_bytes, # ZIP file content
|
| 171 |
+
file_name="my-skill.zip",
|
| 172 |
+
file_type="application/zip",
|
| 173 |
+
),
|
| 174 |
+
user_id="user_123"
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
# List skills
|
| 178 |
+
skills = await LiteLLMSkillsHandler.list_skills(limit=10, offset=0)
|
| 179 |
+
|
| 180 |
+
# Get skill
|
| 181 |
+
skill = await LiteLLMSkillsHandler.get_skill(skill_id="skill_abc123")
|
| 182 |
+
|
| 183 |
+
# Delete skill
|
| 184 |
+
await LiteLLMSkillsHandler.delete_skill(skill_id="skill_abc123")
|
| 185 |
+
```
|
| 186 |
+
|
| 187 |
+
### 2. `transformation.py` - LiteLLMSkillsTransformationHandler
|
| 188 |
+
|
| 189 |
+
SDK-level transformation layer that wraps handler operations:
|
| 190 |
+
|
| 191 |
+
```python
|
| 192 |
+
from litellm.llms.litellm_proxy.skills import LiteLLMSkillsTransformationHandler
|
| 193 |
+
|
| 194 |
+
handler = LiteLLMSkillsTransformationHandler()
|
| 195 |
+
|
| 196 |
+
# Async create
|
| 197 |
+
skill = await handler.create_skill_handler(
|
| 198 |
+
display_title="My Skill",
|
| 199 |
+
files=[zip_file],
|
| 200 |
+
_is_async=True
|
| 201 |
+
)
|
| 202 |
+
```
|
| 203 |
+
|
| 204 |
+
## Skill ZIP Format
|
| 205 |
+
|
| 206 |
+
Skills must be packaged as ZIP files with a `SKILL.md` file:
|
| 207 |
+
|
| 208 |
+
```
|
| 209 |
+
my-skill.zip
|
| 210 |
+
└── my-skill/
|
| 211 |
+
└── SKILL.md
|
| 212 |
+
```
|
| 213 |
+
|
| 214 |
+
### SKILL.md Format
|
| 215 |
+
|
| 216 |
+
```markdown
|
| 217 |
+
---
|
| 218 |
+
name: my-skill
|
| 219 |
+
description: A brief description of what this skill does
|
| 220 |
+
---
|
| 221 |
+
|
| 222 |
+
# My Skill
|
| 223 |
+
|
| 224 |
+
Detailed instructions for the LLM on how to use this skill.
|
| 225 |
+
|
| 226 |
+
## Usage
|
| 227 |
+
|
| 228 |
+
When the user asks about X, use this skill to...
|
| 229 |
+
|
| 230 |
+
## Examples
|
| 231 |
+
|
| 232 |
+
- Example 1: ...
|
| 233 |
+
- Example 2: ...
|
| 234 |
+
```
|
| 235 |
+
|
| 236 |
+
## SDK Usage
|
| 237 |
+
|
| 238 |
+
### Create Skill in LiteLLM Database
|
| 239 |
+
|
| 240 |
+
```python
|
| 241 |
+
import litellm
|
| 242 |
+
|
| 243 |
+
# Create skill stored in LiteLLM DB
|
| 244 |
+
skill = litellm.create_skill(
|
| 245 |
+
display_title="Data Analysis Skill",
|
| 246 |
+
files=[open("data-analysis.zip", "rb")],
|
| 247 |
+
custom_llm_provider="litellm_proxy", # Store in LiteLLM DB
|
| 248 |
+
)
|
| 249 |
+
|
| 250 |
+
print(f"Created skill: {skill.id}") # skill_abc123
|
| 251 |
+
```
|
| 252 |
+
|
| 253 |
+
### Use Skill with Any Provider
|
| 254 |
+
|
| 255 |
+
```python
|
| 256 |
+
import litellm
|
| 257 |
+
|
| 258 |
+
# Use LiteLLM-stored skill with Bedrock
|
| 259 |
+
response = litellm.completion(
|
| 260 |
+
model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
|
| 261 |
+
messages=[{"role": "user", "content": "Analyze this data..."}],
|
| 262 |
+
container={
|
| 263 |
+
"skills": [
|
| 264 |
+
{"type": "custom", "skill_id": "litellm:skill_abc123"} # litellm: prefix
|
| 265 |
+
]
|
| 266 |
+
}
|
| 267 |
+
)
|
| 268 |
+
```
|
| 269 |
+
|
| 270 |
+
## How Skill Resolution Works
|
| 271 |
+
|
| 272 |
+
### Step 1: Request with Skills
|
| 273 |
+
|
| 274 |
+
```python
|
| 275 |
+
{
|
| 276 |
+
"model": "bedrock/claude-3-sonnet",
|
| 277 |
+
"messages": [{"role": "user", "content": "Help me analyze data"}],
|
| 278 |
+
"container": {
|
| 279 |
+
"skills": [
|
| 280 |
+
{"type": "custom", "skill_id": "litellm:skill_abc123"}
|
| 281 |
+
]
|
| 282 |
+
}
|
| 283 |
+
}
|
| 284 |
+
```
|
| 285 |
+
|
| 286 |
+
### Step 2: SkillsInjectionHook Processing
|
| 287 |
+
|
| 288 |
+
The hook (`litellm/proxy/hooks/litellm_skills/main.py`) intercepts the request:
|
| 289 |
+
|
| 290 |
+
1. **Detects `litellm:` prefix** → Fetches skill from database
|
| 291 |
+
2. **Checks model provider** → Bedrock is not Anthropic
|
| 292 |
+
3. **Extracts SKILL.md** from stored ZIP file
|
| 293 |
+
4. **Converts skill to tool** + **Injects content into system prompt**
|
| 294 |
+
|
| 295 |
+
### Step 3: Transformed Request
|
| 296 |
+
|
| 297 |
+
```python
|
| 298 |
+
{
|
| 299 |
+
"model": "bedrock/claude-3-sonnet",
|
| 300 |
+
"messages": [
|
| 301 |
+
{
|
| 302 |
+
"role": "system",
|
| 303 |
+
"content": """
|
| 304 |
+
---
|
| 305 |
+
|
| 306 |
+
# Available Skills
|
| 307 |
+
|
| 308 |
+
## Skill: Data Analysis Skill
|
| 309 |
+
|
| 310 |
+
# Data Analysis Skill
|
| 311 |
+
|
| 312 |
+
This skill helps with data analysis tasks...
|
| 313 |
+
|
| 314 |
+
## Usage
|
| 315 |
+
When the user asks about data analysis...
|
| 316 |
+
"""
|
| 317 |
+
},
|
| 318 |
+
{"role": "user", "content": "Help me analyze data"}
|
| 319 |
+
],
|
| 320 |
+
"tools": [
|
| 321 |
+
{
|
| 322 |
+
"type": "function",
|
| 323 |
+
"function": {
|
| 324 |
+
"name": "skill_abc123",
|
| 325 |
+
"description": "This skill helps with data analysis tasks...",
|
| 326 |
+
"parameters": {"type": "object", "properties": {}, "required": []}
|
| 327 |
+
}
|
| 328 |
+
}
|
| 329 |
+
]
|
| 330 |
+
# container is removed for non-Anthropic providers
|
| 331 |
+
}
|
| 332 |
+
```
|
| 333 |
+
|
| 334 |
+
## Database Schema
|
| 335 |
+
|
| 336 |
+
Skills are stored in `LiteLLM_SkillsTable`:
|
| 337 |
+
|
| 338 |
+
```prisma
|
| 339 |
+
model LiteLLM_SkillsTable {
|
| 340 |
+
skill_id String @id @default(uuid())
|
| 341 |
+
display_title String?
|
| 342 |
+
description String?
|
| 343 |
+
instructions String?
|
| 344 |
+
source String @default("custom")
|
| 345 |
+
latest_version String?
|
| 346 |
+
metadata Json? @default("{}")
|
| 347 |
+
file_content Bytes? // ZIP file binary content
|
| 348 |
+
file_name String? // Original filename
|
| 349 |
+
file_type String? // MIME type
|
| 350 |
+
created_at DateTime @default(now())
|
| 351 |
+
created_by String?
|
| 352 |
+
updated_at DateTime @default(now()) @updatedAt
|
| 353 |
+
updated_by String?
|
| 354 |
+
}
|
| 355 |
+
```
|
| 356 |
+
|
| 357 |
+
## Routing Summary
|
| 358 |
+
|
| 359 |
+
| Scenario | custom_llm_provider | skill_id Format | Behavior |
|
| 360 |
+
|----------|---------------------|-----------------|----------|
|
| 361 |
+
| Create skill on Anthropic | `anthropic` | N/A | Forward to Anthropic API |
|
| 362 |
+
| Create skill in LiteLLM DB | `litellm_proxy` | N/A | Store in database |
|
| 363 |
+
| Use Anthropic native skill | N/A | `skill_xyz` | Pass to Anthropic container.skills |
|
| 364 |
+
| Use LiteLLM skill on Anthropic | N/A | `litellm:skill_abc` | Convert to tools |
|
| 365 |
+
| Use LiteLLM skill on Bedrock/OpenAI | N/A | `litellm:skill_abc` | Convert to tools + inject SKILL.md |
|
| 366 |
+
|
| 367 |
+
## Testing
|
| 368 |
+
|
| 369 |
+
Run the tests:
|
| 370 |
+
|
| 371 |
+
```bash
|
| 372 |
+
pytest tests/proxy_unit_tests/test_skills_db.py -v
|
| 373 |
+
```
|
| 374 |
+
|
| 375 |
+
Tests cover:
|
| 376 |
+
- Creating skills with file content
|
| 377 |
+
- Listing and retrieving skills
|
| 378 |
+
- Deleting skills
|
| 379 |
+
- Hook resolution with ZIP file extraction
|
| 380 |
+
- System prompt injection for non-Anthropic models
|
| 381 |
+
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/__init__.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
LiteLLM Proxy Skills - Database-backed skills storage and execution
|
| 3 |
+
|
| 4 |
+
This module provides:
|
| 5 |
+
- Database-backed skills storage (alternative to Anthropic's cloud-based skills API)
|
| 6 |
+
- Skill content extraction and prompt injection
|
| 7 |
+
- Sandboxed code execution for skills
|
| 8 |
+
- Automatic code execution handler
|
| 9 |
+
|
| 10 |
+
Main components:
|
| 11 |
+
- handler.py: LiteLLMSkillsHandler - database CRUD operations
|
| 12 |
+
- transformation.py: LiteLLMSkillsTransformationHandler - SDK transformation layer
|
| 13 |
+
- prompt_injection.py: SkillPromptInjectionHandler - SKILL.md extraction and injection
|
| 14 |
+
- sandbox_executor.py: SkillsSandboxExecutor - Docker sandbox execution
|
| 15 |
+
- code_execution.py: CodeExecutionHandler - automatic agentic loop
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
from litellm.llms.litellm_proxy.skills.code_execution import (
|
| 19 |
+
LITELLM_CODE_EXECUTION_TOOL,
|
| 20 |
+
CodeExecutionHandler,
|
| 21 |
+
LiteLLMInternalTools,
|
| 22 |
+
add_code_execution_tool,
|
| 23 |
+
code_execution_handler,
|
| 24 |
+
get_litellm_code_execution_tool,
|
| 25 |
+
has_code_execution_tool,
|
| 26 |
+
)
|
| 27 |
+
from litellm.llms.litellm_proxy.skills.constants import (
|
| 28 |
+
DEFAULT_MAX_ITERATIONS,
|
| 29 |
+
DEFAULT_SANDBOX_TIMEOUT,
|
| 30 |
+
)
|
| 31 |
+
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
|
| 32 |
+
from litellm.llms.litellm_proxy.skills.prompt_injection import (
|
| 33 |
+
SkillPromptInjectionHandler,
|
| 34 |
+
)
|
| 35 |
+
from litellm.llms.litellm_proxy.skills.sandbox_executor import SkillsSandboxExecutor
|
| 36 |
+
from litellm.llms.litellm_proxy.skills.transformation import (
|
| 37 |
+
LiteLLMSkillsTransformationHandler,
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
__all__ = [
|
| 41 |
+
"LiteLLMSkillsHandler",
|
| 42 |
+
"LiteLLMSkillsTransformationHandler",
|
| 43 |
+
"SkillPromptInjectionHandler",
|
| 44 |
+
"SkillsSandboxExecutor",
|
| 45 |
+
"CodeExecutionHandler",
|
| 46 |
+
"LiteLLMInternalTools",
|
| 47 |
+
"LITELLM_CODE_EXECUTION_TOOL",
|
| 48 |
+
"get_litellm_code_execution_tool",
|
| 49 |
+
"code_execution_handler",
|
| 50 |
+
"has_code_execution_tool",
|
| 51 |
+
"add_code_execution_tool",
|
| 52 |
+
"DEFAULT_MAX_ITERATIONS",
|
| 53 |
+
"DEFAULT_SANDBOX_TIMEOUT",
|
| 54 |
+
]
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/code_execution.py
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Automatic Code Execution Handler for LiteLLM Skills
|
| 3 |
+
|
| 4 |
+
When `litellm_code_execution` tool is present, this handler automatically:
|
| 5 |
+
1. Makes the LLM call
|
| 6 |
+
2. Executes any code the model generates
|
| 7 |
+
3. Continues the conversation with results
|
| 8 |
+
4. Returns final response with generated files inline (base64)
|
| 9 |
+
|
| 10 |
+
This mimics Anthropic's behavior where code execution happens automatically.
|
| 11 |
+
Generated files are returned directly in the response - no separate storage needed.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import base64
|
| 15 |
+
import json
|
| 16 |
+
from enum import Enum
|
| 17 |
+
from typing import Any, Dict, List, Optional
|
| 18 |
+
|
| 19 |
+
from litellm._logging import verbose_logger
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class LiteLLMInternalTools(str, Enum):
|
| 23 |
+
"""
|
| 24 |
+
Enum for internal LiteLLM tools that are injected into requests.
|
| 25 |
+
|
| 26 |
+
These tools are handled automatically by LiteLLM hooks and are not
|
| 27 |
+
passed to the underlying LLM provider directly.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
CODE_EXECUTION = "litellm_code_execution"
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def get_litellm_code_execution_tool() -> Dict[str, Any]:
|
| 34 |
+
"""
|
| 35 |
+
Returns the litellm_code_execution tool definition in OpenAI format.
|
| 36 |
+
|
| 37 |
+
This tool enables automatic code execution in a sandboxed environment
|
| 38 |
+
when skills include executable Python code.
|
| 39 |
+
"""
|
| 40 |
+
return {
|
| 41 |
+
"type": "function",
|
| 42 |
+
"function": {
|
| 43 |
+
"name": LiteLLMInternalTools.CODE_EXECUTION.value,
|
| 44 |
+
"description": "Execute Python code in a sandboxed environment. Use this to run code that generates files, processes data, or performs computations. Generated files will be returned directly.",
|
| 45 |
+
"parameters": {
|
| 46 |
+
"type": "object",
|
| 47 |
+
"properties": {
|
| 48 |
+
"code": {"type": "string", "description": "Python code to execute"}
|
| 49 |
+
},
|
| 50 |
+
"required": ["code"],
|
| 51 |
+
},
|
| 52 |
+
},
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def get_litellm_code_execution_tool_anthropic() -> Dict[str, Any]:
|
| 57 |
+
"""
|
| 58 |
+
Returns the litellm_code_execution tool definition in Anthropic/messages API format.
|
| 59 |
+
|
| 60 |
+
This tool enables automatic code execution in a sandboxed environment
|
| 61 |
+
when skills include executable Python code.
|
| 62 |
+
"""
|
| 63 |
+
return {
|
| 64 |
+
"name": LiteLLMInternalTools.CODE_EXECUTION.value,
|
| 65 |
+
"description": "Execute Python code in a sandboxed environment. Use this to run code that generates files, processes data, or performs computations. Generated files will be returned directly.",
|
| 66 |
+
"input_schema": {
|
| 67 |
+
"type": "object",
|
| 68 |
+
"properties": {
|
| 69 |
+
"code": {"type": "string", "description": "Python code to execute"}
|
| 70 |
+
},
|
| 71 |
+
"required": ["code"],
|
| 72 |
+
},
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
# Singleton tool definition for backwards compatibility
|
| 77 |
+
LITELLM_CODE_EXECUTION_TOOL = get_litellm_code_execution_tool()
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
class CodeExecutionHandler:
|
| 81 |
+
"""
|
| 82 |
+
Handles automatic code execution for LiteLLM skills.
|
| 83 |
+
|
| 84 |
+
When enabled, this handler intercepts LLM responses with code execution
|
| 85 |
+
tool calls, executes them in a sandbox, and continues the conversation
|
| 86 |
+
automatically until completion.
|
| 87 |
+
"""
|
| 88 |
+
|
| 89 |
+
def __init__(
|
| 90 |
+
self,
|
| 91 |
+
max_iterations: Optional[int] = None,
|
| 92 |
+
sandbox_timeout: Optional[int] = None,
|
| 93 |
+
):
|
| 94 |
+
from litellm.llms.litellm_proxy.skills.constants import (
|
| 95 |
+
DEFAULT_MAX_ITERATIONS,
|
| 96 |
+
DEFAULT_SANDBOX_TIMEOUT,
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
self.max_iterations = max_iterations or DEFAULT_MAX_ITERATIONS
|
| 100 |
+
self.sandbox_timeout = sandbox_timeout or DEFAULT_SANDBOX_TIMEOUT
|
| 101 |
+
|
| 102 |
+
async def execute_with_code_execution(
|
| 103 |
+
self,
|
| 104 |
+
model: str,
|
| 105 |
+
messages: List[Dict],
|
| 106 |
+
tools: List[Dict],
|
| 107 |
+
skill_files: Dict[str, bytes],
|
| 108 |
+
skill_id: Optional[str] = None,
|
| 109 |
+
**kwargs,
|
| 110 |
+
) -> Dict[str, Any]:
|
| 111 |
+
"""
|
| 112 |
+
Execute an LLM call with automatic code execution handling.
|
| 113 |
+
|
| 114 |
+
This method:
|
| 115 |
+
1. Makes the initial LLM call
|
| 116 |
+
2. If model calls litellm_code_execution, executes the code
|
| 117 |
+
3. Continues conversation with results
|
| 118 |
+
4. Repeats until model stops calling tools
|
| 119 |
+
5. Returns final response with generated files inline
|
| 120 |
+
|
| 121 |
+
Args:
|
| 122 |
+
model: Model to use
|
| 123 |
+
messages: Initial messages
|
| 124 |
+
tools: Tools including litellm_code_execution
|
| 125 |
+
skill_files: Dict of skill files for execution
|
| 126 |
+
skill_id: Optional skill ID for tracking
|
| 127 |
+
**kwargs: Additional args for litellm.acompletion
|
| 128 |
+
|
| 129 |
+
Returns:
|
| 130 |
+
Dict with:
|
| 131 |
+
- response: Final LLM response
|
| 132 |
+
- files: List of generated files with content (base64)
|
| 133 |
+
- execution_results: List of code execution results
|
| 134 |
+
"""
|
| 135 |
+
import litellm
|
| 136 |
+
from litellm.llms.litellm_proxy.skills.sandbox_executor import (
|
| 137 |
+
SkillsSandboxExecutor,
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
current_messages = list(messages)
|
| 141 |
+
generated_files: List[Dict[str, Any]] = [] # Files returned directly
|
| 142 |
+
execution_results: List[Dict] = []
|
| 143 |
+
|
| 144 |
+
executor = SkillsSandboxExecutor(timeout=self.sandbox_timeout)
|
| 145 |
+
response: Any = None # Initialize to avoid possibly unbound error
|
| 146 |
+
|
| 147 |
+
for iteration in range(self.max_iterations):
|
| 148 |
+
verbose_logger.debug(
|
| 149 |
+
f"CodeExecutionHandler: Iteration {iteration + 1}/{self.max_iterations}"
|
| 150 |
+
)
|
| 151 |
+
|
| 152 |
+
# Make LLM call
|
| 153 |
+
response = await litellm.acompletion(
|
| 154 |
+
model=model,
|
| 155 |
+
messages=current_messages,
|
| 156 |
+
tools=tools,
|
| 157 |
+
**kwargs,
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
assistant_message = response.choices[0].message # type: ignore
|
| 161 |
+
stop_reason = response.choices[0].finish_reason # type: ignore
|
| 162 |
+
|
| 163 |
+
# Build assistant message for conversation history
|
| 164 |
+
assistant_msg_dict: Dict[str, Any] = {
|
| 165 |
+
"role": "assistant",
|
| 166 |
+
"content": assistant_message.content,
|
| 167 |
+
}
|
| 168 |
+
if assistant_message.tool_calls:
|
| 169 |
+
assistant_msg_dict["tool_calls"] = [
|
| 170 |
+
{
|
| 171 |
+
"id": tc.id,
|
| 172 |
+
"type": "function",
|
| 173 |
+
"function": {
|
| 174 |
+
"name": tc.function.name,
|
| 175 |
+
"arguments": tc.function.arguments,
|
| 176 |
+
},
|
| 177 |
+
}
|
| 178 |
+
for tc in assistant_message.tool_calls
|
| 179 |
+
]
|
| 180 |
+
current_messages.append(assistant_msg_dict)
|
| 181 |
+
|
| 182 |
+
# Check if we're done (no tool calls or not tool_calls finish reason)
|
| 183 |
+
if stop_reason != "tool_calls" or not assistant_message.tool_calls:
|
| 184 |
+
verbose_logger.debug(
|
| 185 |
+
f"CodeExecutionHandler: Completed after {iteration + 1} iterations"
|
| 186 |
+
)
|
| 187 |
+
return {
|
| 188 |
+
"response": response,
|
| 189 |
+
"files": generated_files, # Files returned directly with base64 content
|
| 190 |
+
"execution_results": execution_results,
|
| 191 |
+
"messages": current_messages,
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
# Handle tool calls
|
| 195 |
+
for tool_call in assistant_message.tool_calls:
|
| 196 |
+
tool_name = tool_call.function.name
|
| 197 |
+
|
| 198 |
+
if tool_name == LiteLLMInternalTools.CODE_EXECUTION.value:
|
| 199 |
+
# Execute code in sandbox
|
| 200 |
+
try:
|
| 201 |
+
args = json.loads(tool_call.function.arguments)
|
| 202 |
+
code = args.get("code", "")
|
| 203 |
+
|
| 204 |
+
verbose_logger.debug(
|
| 205 |
+
f"CodeExecutionHandler: Executing code ({len(code)} chars)"
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
exec_result = executor.execute(
|
| 209 |
+
code=code,
|
| 210 |
+
skill_files=skill_files,
|
| 211 |
+
)
|
| 212 |
+
|
| 213 |
+
verbose_logger.debug(
|
| 214 |
+
f"CodeExecutionHandler: Execution result: {exec_result}"
|
| 215 |
+
)
|
| 216 |
+
|
| 217 |
+
execution_results.append(
|
| 218 |
+
{
|
| 219 |
+
"iteration": iteration,
|
| 220 |
+
"success": exec_result["success"],
|
| 221 |
+
"output": exec_result["output"],
|
| 222 |
+
"error": exec_result["error"],
|
| 223 |
+
"files": [f["name"] for f in exec_result["files"]],
|
| 224 |
+
}
|
| 225 |
+
)
|
| 226 |
+
|
| 227 |
+
# Build tool result content
|
| 228 |
+
tool_result = exec_result["output"] or ""
|
| 229 |
+
|
| 230 |
+
# Collect generated files (returned directly, no storage)
|
| 231 |
+
if exec_result["files"]:
|
| 232 |
+
tool_result += "\n\nGenerated files:"
|
| 233 |
+
for f in exec_result["files"]:
|
| 234 |
+
file_content = base64.b64decode(f["content_base64"])
|
| 235 |
+
# Add to generated files list (returned in response)
|
| 236 |
+
generated_files.append(
|
| 237 |
+
{
|
| 238 |
+
"name": f["name"],
|
| 239 |
+
"mime_type": f["mime_type"],
|
| 240 |
+
"content_base64": f["content_base64"],
|
| 241 |
+
"size": len(file_content),
|
| 242 |
+
}
|
| 243 |
+
)
|
| 244 |
+
tool_result += (
|
| 245 |
+
f"\n- {f['name']} ({len(file_content)} bytes)"
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
verbose_logger.debug(
|
| 249 |
+
f"CodeExecutionHandler: Generated file {f['name']} ({len(file_content)} bytes)"
|
| 250 |
+
)
|
| 251 |
+
|
| 252 |
+
if exec_result["error"]:
|
| 253 |
+
tool_result += f"\n\nError:\n{exec_result['error']}"
|
| 254 |
+
|
| 255 |
+
except Exception as e:
|
| 256 |
+
tool_result = f"Code execution failed: {str(e)}"
|
| 257 |
+
execution_results.append(
|
| 258 |
+
{
|
| 259 |
+
"iteration": iteration,
|
| 260 |
+
"success": False,
|
| 261 |
+
"error": str(e),
|
| 262 |
+
}
|
| 263 |
+
)
|
| 264 |
+
|
| 265 |
+
# Add tool result to messages
|
| 266 |
+
current_messages.append(
|
| 267 |
+
{
|
| 268 |
+
"role": "tool",
|
| 269 |
+
"tool_call_id": tool_call.id,
|
| 270 |
+
"content": tool_result,
|
| 271 |
+
}
|
| 272 |
+
)
|
| 273 |
+
else:
|
| 274 |
+
# Non-code-execution tool - pass through
|
| 275 |
+
# In a full implementation, this would call other tool handlers
|
| 276 |
+
current_messages.append(
|
| 277 |
+
{
|
| 278 |
+
"role": "tool",
|
| 279 |
+
"tool_call_id": tool_call.id,
|
| 280 |
+
"content": f"Tool '{tool_name}' not handled by code execution handler",
|
| 281 |
+
}
|
| 282 |
+
)
|
| 283 |
+
|
| 284 |
+
# Max iterations reached
|
| 285 |
+
verbose_logger.warning(
|
| 286 |
+
f"CodeExecutionHandler: Max iterations ({self.max_iterations}) reached"
|
| 287 |
+
)
|
| 288 |
+
return {
|
| 289 |
+
"response": response,
|
| 290 |
+
"files": generated_files,
|
| 291 |
+
"execution_results": execution_results,
|
| 292 |
+
"messages": current_messages,
|
| 293 |
+
"max_iterations_reached": True,
|
| 294 |
+
}
|
| 295 |
+
|
| 296 |
+
|
| 297 |
+
def has_code_execution_tool(tools: Optional[List[Dict]]) -> bool:
|
| 298 |
+
"""Check if litellm_code_execution tool is in the tools list."""
|
| 299 |
+
if not tools:
|
| 300 |
+
return False
|
| 301 |
+
for tool in tools:
|
| 302 |
+
func = tool.get("function", {})
|
| 303 |
+
if func.get("name") == LiteLLMInternalTools.CODE_EXECUTION.value:
|
| 304 |
+
return True
|
| 305 |
+
return False
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def add_code_execution_tool(tools: Optional[List[Dict]]) -> List[Dict]:
|
| 309 |
+
"""Add litellm_code_execution tool if not already present."""
|
| 310 |
+
tools = tools or []
|
| 311 |
+
if not has_code_execution_tool(tools):
|
| 312 |
+
tools.append(LITELLM_CODE_EXECUTION_TOOL)
|
| 313 |
+
return tools
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
# Global handler instance
|
| 317 |
+
code_execution_handler = CodeExecutionHandler()
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/constants.py
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Constants for LiteLLM Skills
|
| 3 |
+
|
| 4 |
+
Centralized constants for skills processing, code execution, and sandbox configuration.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
# Code execution loop settings
|
| 8 |
+
DEFAULT_MAX_ITERATIONS: int = 10
|
| 9 |
+
"""Maximum number of iterations for the automatic code execution loop."""
|
| 10 |
+
|
| 11 |
+
DEFAULT_SANDBOX_TIMEOUT: int = 120
|
| 12 |
+
"""Default timeout in seconds for sandbox code execution."""
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/handler.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Handler for LiteLLM database-backed skills operations.
|
| 3 |
+
|
| 4 |
+
This module contains the actual database operations for skills CRUD.
|
| 5 |
+
Used by the transformation layer and skills injection hook.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import uuid
|
| 9 |
+
from typing import Any, Dict, List, Optional
|
| 10 |
+
|
| 11 |
+
from litellm._logging import verbose_logger
|
| 12 |
+
from litellm.proxy._types import LiteLLM_SkillsTable, NewSkillRequest
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _prisma_skill_to_litellm(prisma_skill) -> LiteLLM_SkillsTable:
|
| 16 |
+
"""
|
| 17 |
+
Convert a Prisma skill record to LiteLLM_SkillsTable.
|
| 18 |
+
|
| 19 |
+
Handles Base64 decoding of file_content field.
|
| 20 |
+
"""
|
| 21 |
+
import base64
|
| 22 |
+
|
| 23 |
+
data = prisma_skill.model_dump()
|
| 24 |
+
|
| 25 |
+
# Decode Base64 file_content back to bytes
|
| 26 |
+
# model_dump() converts Base64 field to base64-encoded string
|
| 27 |
+
if data.get("file_content") is not None:
|
| 28 |
+
if isinstance(data["file_content"], str):
|
| 29 |
+
data["file_content"] = base64.b64decode(data["file_content"])
|
| 30 |
+
elif isinstance(data["file_content"], bytes):
|
| 31 |
+
# Already bytes, no conversion needed
|
| 32 |
+
pass
|
| 33 |
+
|
| 34 |
+
return LiteLLM_SkillsTable(**data)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class LiteLLMSkillsHandler:
|
| 38 |
+
"""
|
| 39 |
+
Handler for LiteLLM database-backed skills operations.
|
| 40 |
+
|
| 41 |
+
This class provides static methods for CRUD operations on skills
|
| 42 |
+
stored in the LiteLLM proxy database (LiteLLM_SkillsTable).
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
@staticmethod
|
| 46 |
+
async def _get_prisma_client():
|
| 47 |
+
"""Get the prisma client from proxy server."""
|
| 48 |
+
from litellm.proxy.proxy_server import prisma_client
|
| 49 |
+
|
| 50 |
+
if prisma_client is None:
|
| 51 |
+
raise ValueError(
|
| 52 |
+
"Prisma client is not initialized. "
|
| 53 |
+
"Database connection required for LiteLLM skills."
|
| 54 |
+
)
|
| 55 |
+
return prisma_client
|
| 56 |
+
|
| 57 |
+
@staticmethod
|
| 58 |
+
async def create_skill(
|
| 59 |
+
data: NewSkillRequest,
|
| 60 |
+
user_id: Optional[str] = None,
|
| 61 |
+
) -> LiteLLM_SkillsTable:
|
| 62 |
+
"""
|
| 63 |
+
Create a new skill in the LiteLLM database.
|
| 64 |
+
|
| 65 |
+
Args:
|
| 66 |
+
data: NewSkillRequest with skill details
|
| 67 |
+
user_id: Optional user ID for tracking
|
| 68 |
+
|
| 69 |
+
Returns:
|
| 70 |
+
LiteLLM_SkillsTable record
|
| 71 |
+
"""
|
| 72 |
+
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
|
| 73 |
+
|
| 74 |
+
skill_id = f"litellm_skill_{uuid.uuid4()}"
|
| 75 |
+
|
| 76 |
+
skill_data: Dict[str, Any] = {
|
| 77 |
+
"skill_id": skill_id,
|
| 78 |
+
"display_title": data.display_title,
|
| 79 |
+
"description": data.description,
|
| 80 |
+
"instructions": data.instructions,
|
| 81 |
+
"source": "custom",
|
| 82 |
+
"created_by": user_id,
|
| 83 |
+
"updated_by": user_id,
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
# Handle metadata
|
| 87 |
+
if data.metadata is not None:
|
| 88 |
+
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
| 89 |
+
|
| 90 |
+
skill_data["metadata"] = safe_dumps(data.metadata)
|
| 91 |
+
|
| 92 |
+
# Handle file content - wrap bytes in Base64 for Prisma
|
| 93 |
+
if data.file_content is not None:
|
| 94 |
+
from prisma.fields import Base64
|
| 95 |
+
|
| 96 |
+
skill_data["file_content"] = Base64.encode(data.file_content)
|
| 97 |
+
if data.file_name is not None:
|
| 98 |
+
skill_data["file_name"] = data.file_name
|
| 99 |
+
if data.file_type is not None:
|
| 100 |
+
skill_data["file_type"] = data.file_type
|
| 101 |
+
|
| 102 |
+
verbose_logger.debug(
|
| 103 |
+
f"LiteLLMSkillsHandler: Creating skill {skill_id} with title={data.display_title}"
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
new_skill = await prisma_client.db.litellm_skillstable.create(data=skill_data)
|
| 107 |
+
|
| 108 |
+
return _prisma_skill_to_litellm(new_skill)
|
| 109 |
+
|
| 110 |
+
@staticmethod
|
| 111 |
+
async def list_skills(
|
| 112 |
+
limit: int = 20,
|
| 113 |
+
offset: int = 0,
|
| 114 |
+
) -> List[LiteLLM_SkillsTable]:
|
| 115 |
+
"""
|
| 116 |
+
List skills from the LiteLLM database.
|
| 117 |
+
|
| 118 |
+
Args:
|
| 119 |
+
limit: Maximum number of skills to return
|
| 120 |
+
offset: Number of skills to skip
|
| 121 |
+
|
| 122 |
+
Returns:
|
| 123 |
+
List of LiteLLM_SkillsTable records
|
| 124 |
+
"""
|
| 125 |
+
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
|
| 126 |
+
|
| 127 |
+
verbose_logger.debug(
|
| 128 |
+
f"LiteLLMSkillsHandler: Listing skills with limit={limit}, offset={offset}"
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
skills = await prisma_client.db.litellm_skillstable.find_many(
|
| 132 |
+
take=limit,
|
| 133 |
+
skip=offset,
|
| 134 |
+
order={"created_at": "desc"},
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
return [_prisma_skill_to_litellm(s) for s in skills]
|
| 138 |
+
|
| 139 |
+
@staticmethod
|
| 140 |
+
async def get_skill(skill_id: str) -> LiteLLM_SkillsTable:
|
| 141 |
+
"""
|
| 142 |
+
Get a skill by ID from the LiteLLM database.
|
| 143 |
+
|
| 144 |
+
Args:
|
| 145 |
+
skill_id: The skill ID to retrieve
|
| 146 |
+
|
| 147 |
+
Returns:
|
| 148 |
+
LiteLLM_SkillsTable record
|
| 149 |
+
|
| 150 |
+
Raises:
|
| 151 |
+
ValueError: If skill not found
|
| 152 |
+
"""
|
| 153 |
+
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
|
| 154 |
+
|
| 155 |
+
verbose_logger.debug(f"LiteLLMSkillsHandler: Getting skill {skill_id}")
|
| 156 |
+
|
| 157 |
+
skill = await prisma_client.db.litellm_skillstable.find_unique(
|
| 158 |
+
where={"skill_id": skill_id}
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
if skill is None:
|
| 162 |
+
raise ValueError(f"Skill not found: {skill_id}")
|
| 163 |
+
|
| 164 |
+
return _prisma_skill_to_litellm(skill)
|
| 165 |
+
|
| 166 |
+
@staticmethod
|
| 167 |
+
async def delete_skill(skill_id: str) -> Dict[str, str]:
|
| 168 |
+
"""
|
| 169 |
+
Delete a skill by ID from the LiteLLM database.
|
| 170 |
+
|
| 171 |
+
Args:
|
| 172 |
+
skill_id: The skill ID to delete
|
| 173 |
+
|
| 174 |
+
Returns:
|
| 175 |
+
Dict with id and type of deleted skill
|
| 176 |
+
|
| 177 |
+
Raises:
|
| 178 |
+
ValueError: If skill not found
|
| 179 |
+
"""
|
| 180 |
+
prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
|
| 181 |
+
|
| 182 |
+
verbose_logger.debug(f"LiteLLMSkillsHandler: Deleting skill {skill_id}")
|
| 183 |
+
|
| 184 |
+
# Check if skill exists
|
| 185 |
+
skill = await prisma_client.db.litellm_skillstable.find_unique(
|
| 186 |
+
where={"skill_id": skill_id}
|
| 187 |
+
)
|
| 188 |
+
|
| 189 |
+
if skill is None:
|
| 190 |
+
raise ValueError(f"Skill not found: {skill_id}")
|
| 191 |
+
|
| 192 |
+
# Delete the skill
|
| 193 |
+
await prisma_client.db.litellm_skillstable.delete(where={"skill_id": skill_id})
|
| 194 |
+
|
| 195 |
+
return {"id": skill_id, "type": "skill_deleted"}
|
| 196 |
+
|
| 197 |
+
@staticmethod
|
| 198 |
+
async def fetch_skill_from_db(skill_id: str) -> Optional[LiteLLM_SkillsTable]:
|
| 199 |
+
"""
|
| 200 |
+
Fetch a skill from the database (used by skills injection hook).
|
| 201 |
+
|
| 202 |
+
This is a convenience method that returns None instead of raising
|
| 203 |
+
an exception if the skill is not found.
|
| 204 |
+
|
| 205 |
+
Args:
|
| 206 |
+
skill_id: The skill ID to fetch
|
| 207 |
+
|
| 208 |
+
Returns:
|
| 209 |
+
LiteLLM_SkillsTable or None if not found
|
| 210 |
+
"""
|
| 211 |
+
try:
|
| 212 |
+
return await LiteLLMSkillsHandler.get_skill(skill_id)
|
| 213 |
+
except ValueError:
|
| 214 |
+
return None
|
| 215 |
+
except Exception as e:
|
| 216 |
+
verbose_logger.warning(
|
| 217 |
+
f"LiteLLMSkillsHandler: Error fetching skill {skill_id}: {e}"
|
| 218 |
+
)
|
| 219 |
+
return None
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/prompt_injection.py
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Prompt Injection Handler for LiteLLM Skills
|
| 3 |
+
|
| 4 |
+
Handles extraction of skill content (SKILL.md) from stored ZIP files
|
| 5 |
+
and injection into the system prompt for non-Anthropic models.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import zipfile
|
| 9 |
+
from io import BytesIO
|
| 10 |
+
from typing import Any, Dict, List, Optional
|
| 11 |
+
|
| 12 |
+
from litellm._logging import verbose_logger
|
| 13 |
+
from litellm.proxy._types import LiteLLM_SkillsTable
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class SkillPromptInjectionHandler:
|
| 17 |
+
"""
|
| 18 |
+
Handles skill content extraction and system prompt injection.
|
| 19 |
+
|
| 20 |
+
Responsibilities:
|
| 21 |
+
- Extract SKILL.md content from skill ZIP files
|
| 22 |
+
- Extract ALL files from ZIP for code execution
|
| 23 |
+
- Inject skill content into system message
|
| 24 |
+
- Create execute_code tool definition
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
def extract_skill_content(self, skill: LiteLLM_SkillsTable) -> Optional[str]:
|
| 28 |
+
"""
|
| 29 |
+
Extract skill content from the stored zip file.
|
| 30 |
+
|
| 31 |
+
Looks for SKILL.md or README.md in the zip and returns its content.
|
| 32 |
+
This content describes the skill's capabilities and instructions.
|
| 33 |
+
|
| 34 |
+
Args:
|
| 35 |
+
skill: The skill from LiteLLM database
|
| 36 |
+
|
| 37 |
+
Returns:
|
| 38 |
+
The skill content as a string, or None if not available
|
| 39 |
+
"""
|
| 40 |
+
if not skill.file_content:
|
| 41 |
+
return skill.instructions
|
| 42 |
+
|
| 43 |
+
try:
|
| 44 |
+
zip_buffer = BytesIO(skill.file_content)
|
| 45 |
+
with zipfile.ZipFile(zip_buffer, "r") as zf:
|
| 46 |
+
# Look for SKILL.md first
|
| 47 |
+
for name in zf.namelist():
|
| 48 |
+
if name.endswith("SKILL.md"):
|
| 49 |
+
content = zf.read(name).decode("utf-8")
|
| 50 |
+
if content:
|
| 51 |
+
return f"## Skill: {skill.display_title or skill.skill_id}\n\n{content}"
|
| 52 |
+
|
| 53 |
+
# Fall back to README.md
|
| 54 |
+
for name in zf.namelist():
|
| 55 |
+
if name.endswith("README.md"):
|
| 56 |
+
content = zf.read(name).decode("utf-8")
|
| 57 |
+
if content:
|
| 58 |
+
return f"## Skill: {skill.display_title or skill.skill_id}\n\n{content}"
|
| 59 |
+
|
| 60 |
+
# Fall back to any .md file
|
| 61 |
+
for name in zf.namelist():
|
| 62 |
+
if name.endswith(".md"):
|
| 63 |
+
content = zf.read(name).decode("utf-8")
|
| 64 |
+
if content:
|
| 65 |
+
return f"## Skill: {skill.display_title or skill.skill_id}\n\n{content}"
|
| 66 |
+
except Exception as e:
|
| 67 |
+
verbose_logger.warning(
|
| 68 |
+
f"SkillPromptInjectionHandler: Error extracting content from skill {skill.skill_id}: {e}"
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
return skill.instructions
|
| 72 |
+
|
| 73 |
+
def extract_all_files(self, skill: LiteLLM_SkillsTable) -> Dict[str, bytes]:
|
| 74 |
+
"""
|
| 75 |
+
Extract ALL files from skill ZIP for code execution.
|
| 76 |
+
|
| 77 |
+
Returns a dict mapping file paths to their binary content.
|
| 78 |
+
The paths have the skill folder prefix removed (e.g., "slack-gif-creator/core/..." -> "core/...").
|
| 79 |
+
|
| 80 |
+
Args:
|
| 81 |
+
skill: The skill from LiteLLM database
|
| 82 |
+
|
| 83 |
+
Returns:
|
| 84 |
+
Dict mapping file paths to binary content
|
| 85 |
+
"""
|
| 86 |
+
files: Dict[str, bytes] = {}
|
| 87 |
+
|
| 88 |
+
if not skill.file_content:
|
| 89 |
+
return files
|
| 90 |
+
|
| 91 |
+
try:
|
| 92 |
+
zip_buffer = BytesIO(skill.file_content)
|
| 93 |
+
with zipfile.ZipFile(zip_buffer, "r") as zf:
|
| 94 |
+
for name in zf.namelist():
|
| 95 |
+
# Skip directories
|
| 96 |
+
if name.endswith("/"):
|
| 97 |
+
continue
|
| 98 |
+
|
| 99 |
+
# Remove skill folder prefix (first path component)
|
| 100 |
+
parts = name.split("/")
|
| 101 |
+
if len(parts) > 1:
|
| 102 |
+
clean_path = "/".join(parts[1:])
|
| 103 |
+
else:
|
| 104 |
+
clean_path = name
|
| 105 |
+
|
| 106 |
+
if clean_path:
|
| 107 |
+
files[clean_path] = zf.read(name)
|
| 108 |
+
except Exception as e:
|
| 109 |
+
verbose_logger.warning(
|
| 110 |
+
f"SkillPromptInjectionHandler: Error extracting files from skill {skill.skill_id}: {e}"
|
| 111 |
+
)
|
| 112 |
+
|
| 113 |
+
return files
|
| 114 |
+
|
| 115 |
+
def inject_skill_content_to_messages(
|
| 116 |
+
self, data: dict, skill_contents: List[str], use_anthropic_format: bool = False
|
| 117 |
+
) -> dict:
|
| 118 |
+
"""
|
| 119 |
+
Inject skill content into the system prompt.
|
| 120 |
+
|
| 121 |
+
For Anthropic messages API (use_anthropic_format=True):
|
| 122 |
+
- Injects into top-level 'system' parameter (not in messages array)
|
| 123 |
+
|
| 124 |
+
For OpenAI-style APIs (use_anthropic_format=False):
|
| 125 |
+
- Injects into messages array with role="system"
|
| 126 |
+
|
| 127 |
+
Args:
|
| 128 |
+
data: The request data dict
|
| 129 |
+
skill_contents: List of skill content strings to inject
|
| 130 |
+
use_anthropic_format: If True, use top-level 'system' param for Anthropic
|
| 131 |
+
|
| 132 |
+
Returns:
|
| 133 |
+
Modified data dict with skill content in system prompt
|
| 134 |
+
"""
|
| 135 |
+
if not skill_contents:
|
| 136 |
+
return data
|
| 137 |
+
|
| 138 |
+
# Build the skill injection text
|
| 139 |
+
skill_section = "\n\n---\n\n# Available Skills\n\n" + "\n\n---\n\n".join(
|
| 140 |
+
skill_contents
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
if use_anthropic_format:
|
| 144 |
+
# Anthropic messages API: use top-level 'system' parameter
|
| 145 |
+
current_system = data.get("system", "")
|
| 146 |
+
if current_system:
|
| 147 |
+
data["system"] = current_system + skill_section
|
| 148 |
+
else:
|
| 149 |
+
data["system"] = skill_section.strip()
|
| 150 |
+
return data
|
| 151 |
+
|
| 152 |
+
# OpenAI-style: inject into messages array
|
| 153 |
+
messages = data.get("messages", [])
|
| 154 |
+
if not messages:
|
| 155 |
+
return data
|
| 156 |
+
|
| 157 |
+
# Find or create system message
|
| 158 |
+
system_msg_idx = None
|
| 159 |
+
for i, msg in enumerate(messages):
|
| 160 |
+
if isinstance(msg, dict) and msg.get("role") == "system":
|
| 161 |
+
system_msg_idx = i
|
| 162 |
+
break
|
| 163 |
+
|
| 164 |
+
if system_msg_idx is not None:
|
| 165 |
+
# Append to existing system message
|
| 166 |
+
current_content = messages[system_msg_idx].get("content", "")
|
| 167 |
+
messages[system_msg_idx]["content"] = current_content + skill_section
|
| 168 |
+
else:
|
| 169 |
+
# Create new system message at the beginning
|
| 170 |
+
messages.insert(0, {"role": "system", "content": skill_section.strip()})
|
| 171 |
+
|
| 172 |
+
data["messages"] = messages
|
| 173 |
+
return data
|
| 174 |
+
|
| 175 |
+
def create_execute_code_tool(self, skill_modules: List[str]) -> Dict[str, Any]:
|
| 176 |
+
"""
|
| 177 |
+
Create the execute_code tool definition.
|
| 178 |
+
|
| 179 |
+
This tool allows the model to execute Python code with access
|
| 180 |
+
to the skill's modules (e.g., 'from core.gif_builder import GIFBuilder').
|
| 181 |
+
|
| 182 |
+
Args:
|
| 183 |
+
skill_modules: List of available module paths (e.g., ["core/gif_builder.py"])
|
| 184 |
+
|
| 185 |
+
Returns:
|
| 186 |
+
OpenAI-style tool definition
|
| 187 |
+
"""
|
| 188 |
+
# Format module list for description
|
| 189 |
+
module_examples = []
|
| 190 |
+
for mod in skill_modules[:5]: # Limit to 5 examples
|
| 191 |
+
if mod.endswith(".py"):
|
| 192 |
+
# Convert path to import: "core/gif_builder.py" -> "from core.gif_builder import ..."
|
| 193 |
+
import_path = mod.replace("/", ".").replace(".py", "")
|
| 194 |
+
module_examples.append(f"from {import_path} import ...")
|
| 195 |
+
|
| 196 |
+
module_hint = ""
|
| 197 |
+
if module_examples:
|
| 198 |
+
module_hint = f" Available modules: {', '.join(module_examples)}"
|
| 199 |
+
|
| 200 |
+
return {
|
| 201 |
+
"type": "function",
|
| 202 |
+
"function": {
|
| 203 |
+
"name": "execute_code",
|
| 204 |
+
"description": f"Execute Python code in a sandboxed environment. Generated files will be returned.{module_hint}",
|
| 205 |
+
"parameters": {
|
| 206 |
+
"type": "object",
|
| 207 |
+
"properties": {
|
| 208 |
+
"code": {
|
| 209 |
+
"type": "string",
|
| 210 |
+
"description": "Python code to execute. You can import skill modules and use standard libraries.",
|
| 211 |
+
}
|
| 212 |
+
},
|
| 213 |
+
"required": ["code"],
|
| 214 |
+
},
|
| 215 |
+
},
|
| 216 |
+
}
|
| 217 |
+
|
| 218 |
+
def convert_skill_to_tool(self, skill: LiteLLM_SkillsTable) -> Dict[str, Any]:
|
| 219 |
+
"""
|
| 220 |
+
Convert a LiteLLM skill to an OpenAI-style tool.
|
| 221 |
+
|
| 222 |
+
The skill's instructions are used as the function description,
|
| 223 |
+
allowing the model to understand when and how to use the skill.
|
| 224 |
+
|
| 225 |
+
Args:
|
| 226 |
+
skill: The skill from LiteLLM database
|
| 227 |
+
|
| 228 |
+
Returns:
|
| 229 |
+
OpenAI-style tool definition
|
| 230 |
+
"""
|
| 231 |
+
# Create a function name from skill_id (sanitize for function naming)
|
| 232 |
+
func_name = skill.skill_id.replace("-", "_").replace(" ", "_")
|
| 233 |
+
|
| 234 |
+
# Use instructions as description, fall back to description or title
|
| 235 |
+
description = (
|
| 236 |
+
skill.instructions
|
| 237 |
+
or skill.description
|
| 238 |
+
or skill.display_title
|
| 239 |
+
or f"Skill: {skill.skill_id}"
|
| 240 |
+
)
|
| 241 |
+
|
| 242 |
+
# Truncate description if too long (OpenAI has limits)
|
| 243 |
+
max_desc_length = 1024
|
| 244 |
+
if len(description) > max_desc_length:
|
| 245 |
+
description = description[: max_desc_length - 3] + "..."
|
| 246 |
+
|
| 247 |
+
tool: Dict[str, Any] = {
|
| 248 |
+
"type": "function",
|
| 249 |
+
"function": {
|
| 250 |
+
"name": func_name,
|
| 251 |
+
"description": description,
|
| 252 |
+
"parameters": {
|
| 253 |
+
"type": "object",
|
| 254 |
+
"properties": {},
|
| 255 |
+
"required": [],
|
| 256 |
+
},
|
| 257 |
+
},
|
| 258 |
+
}
|
| 259 |
+
|
| 260 |
+
# If skill has metadata with parameter definitions, use them
|
| 261 |
+
if skill.metadata and isinstance(skill.metadata, dict):
|
| 262 |
+
params = skill.metadata.get("parameters")
|
| 263 |
+
if params and isinstance(params, dict):
|
| 264 |
+
tool["function"]["parameters"] = params
|
| 265 |
+
|
| 266 |
+
return tool
|
| 267 |
+
|
| 268 |
+
def convert_skill_to_anthropic_tool(
|
| 269 |
+
self, skill: LiteLLM_SkillsTable
|
| 270 |
+
) -> Dict[str, Any]:
|
| 271 |
+
"""
|
| 272 |
+
Convert a LiteLLM skill to an Anthropic-style tool (messages API format).
|
| 273 |
+
|
| 274 |
+
Args:
|
| 275 |
+
skill: The skill from LiteLLM database
|
| 276 |
+
|
| 277 |
+
Returns:
|
| 278 |
+
Anthropic-style tool definition with name, description, input_schema
|
| 279 |
+
"""
|
| 280 |
+
func_name = skill.skill_id.replace("-", "_").replace(" ", "_")
|
| 281 |
+
|
| 282 |
+
description = (
|
| 283 |
+
skill.instructions
|
| 284 |
+
or skill.description
|
| 285 |
+
or skill.display_title
|
| 286 |
+
or f"Skill: {skill.skill_id}"
|
| 287 |
+
)
|
| 288 |
+
|
| 289 |
+
max_desc_length = 1024
|
| 290 |
+
if len(description) > max_desc_length:
|
| 291 |
+
description = description[: max_desc_length - 3] + "..."
|
| 292 |
+
|
| 293 |
+
input_schema: Dict[str, Any] = {
|
| 294 |
+
"type": "object",
|
| 295 |
+
"properties": {},
|
| 296 |
+
"required": [],
|
| 297 |
+
}
|
| 298 |
+
|
| 299 |
+
if skill.metadata and isinstance(skill.metadata, dict):
|
| 300 |
+
params = skill.metadata.get("parameters")
|
| 301 |
+
if params and isinstance(params, dict):
|
| 302 |
+
input_schema = params
|
| 303 |
+
|
| 304 |
+
return {
|
| 305 |
+
"name": func_name,
|
| 306 |
+
"description": description,
|
| 307 |
+
"input_schema": input_schema,
|
| 308 |
+
}
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/sandbox_executor.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Sandbox Executor for LiteLLM Skills
|
| 3 |
+
|
| 4 |
+
Executes skill code in a sandboxed environment using llm-sandbox.
|
| 5 |
+
Supports Docker, Podman, and Kubernetes backends.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import base64
|
| 9 |
+
import os
|
| 10 |
+
from typing import Any, Dict, List, Optional
|
| 11 |
+
|
| 12 |
+
from litellm._logging import verbose_logger
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class SkillsSandboxExecutor:
|
| 16 |
+
"""
|
| 17 |
+
Executes skill code in llm-sandbox Docker container.
|
| 18 |
+
|
| 19 |
+
Responsibilities:
|
| 20 |
+
- Create sandbox session with skill files
|
| 21 |
+
- Install requirements
|
| 22 |
+
- Execute model-generated code
|
| 23 |
+
- Collect generated files (GIFs, images, etc.)
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
def __init__(
|
| 27 |
+
self,
|
| 28 |
+
timeout: int = 60,
|
| 29 |
+
backend: str = "docker",
|
| 30 |
+
image: Optional[str] = None,
|
| 31 |
+
):
|
| 32 |
+
"""
|
| 33 |
+
Initialize the sandbox executor.
|
| 34 |
+
|
| 35 |
+
Args:
|
| 36 |
+
timeout: Maximum execution time in seconds
|
| 37 |
+
backend: Sandbox backend ("docker", "podman", "kubernetes")
|
| 38 |
+
image: Custom Docker image (default: uses llm-sandbox default)
|
| 39 |
+
"""
|
| 40 |
+
self.timeout = timeout
|
| 41 |
+
self.backend = backend
|
| 42 |
+
self.image = image
|
| 43 |
+
self._session = None
|
| 44 |
+
|
| 45 |
+
def execute(
|
| 46 |
+
self,
|
| 47 |
+
code: str,
|
| 48 |
+
skill_files: Dict[str, bytes],
|
| 49 |
+
requirements: Optional[str] = None,
|
| 50 |
+
) -> Dict[str, Any]:
|
| 51 |
+
"""
|
| 52 |
+
Execute code with skill files in sandbox.
|
| 53 |
+
|
| 54 |
+
Args:
|
| 55 |
+
code: Python code to execute
|
| 56 |
+
skill_files: Dict mapping file paths to binary content
|
| 57 |
+
requirements: Optional requirements.txt content
|
| 58 |
+
|
| 59 |
+
Returns:
|
| 60 |
+
{
|
| 61 |
+
"success": bool,
|
| 62 |
+
"output": str,
|
| 63 |
+
"error": str (if failed),
|
| 64 |
+
"files": [{"name": str, "content_base64": str, "mime_type": str}]
|
| 65 |
+
}
|
| 66 |
+
"""
|
| 67 |
+
try:
|
| 68 |
+
from llm_sandbox import SandboxSession
|
| 69 |
+
except ImportError:
|
| 70 |
+
verbose_logger.error(
|
| 71 |
+
"SkillsSandboxExecutor: llm-sandbox not installed. "
|
| 72 |
+
"Install with: pip install llm-sandbox"
|
| 73 |
+
)
|
| 74 |
+
return {
|
| 75 |
+
"success": False,
|
| 76 |
+
"output": "",
|
| 77 |
+
"error": "llm-sandbox not installed. Install with: pip install llm-sandbox",
|
| 78 |
+
"files": [],
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
try:
|
| 82 |
+
# Create sandbox session
|
| 83 |
+
session_kwargs: Dict[str, Any] = {
|
| 84 |
+
"lang": "python",
|
| 85 |
+
"verbose": False,
|
| 86 |
+
}
|
| 87 |
+
|
| 88 |
+
if self.image:
|
| 89 |
+
session_kwargs["image"] = self.image
|
| 90 |
+
|
| 91 |
+
with SandboxSession(**session_kwargs) as session:
|
| 92 |
+
# 1. Copy skill files into sandbox using copy_to_runtime
|
| 93 |
+
import tempfile
|
| 94 |
+
|
| 95 |
+
# Create a temp directory to stage files
|
| 96 |
+
with tempfile.TemporaryDirectory() as tmpdir:
|
| 97 |
+
for path, content in skill_files.items():
|
| 98 |
+
# Create the file in temp directory
|
| 99 |
+
local_path = os.path.join(tmpdir, path)
|
| 100 |
+
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
| 101 |
+
with open(local_path, "wb") as f:
|
| 102 |
+
f.write(content)
|
| 103 |
+
|
| 104 |
+
# Copy to sandbox
|
| 105 |
+
sandbox_path = f"/sandbox/{path}"
|
| 106 |
+
session.copy_to_runtime(local_path, sandbox_path)
|
| 107 |
+
|
| 108 |
+
verbose_logger.debug(
|
| 109 |
+
f"SkillsSandboxExecutor: Copied {len(skill_files)} files to sandbox"
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
# 2. Install requirements if present
|
| 113 |
+
req_packages = None
|
| 114 |
+
if requirements:
|
| 115 |
+
req_packages = requirements.strip().replace("\n", " ")
|
| 116 |
+
elif "requirements.txt" in skill_files:
|
| 117 |
+
req_content = skill_files["requirements.txt"].decode("utf-8")
|
| 118 |
+
req_packages = req_content.strip().replace("\n", " ")
|
| 119 |
+
|
| 120 |
+
if req_packages:
|
| 121 |
+
# Run pip install as code
|
| 122 |
+
pip_code = f"""
|
| 123 |
+
import subprocess
|
| 124 |
+
subprocess.run(['pip', 'install'] + '{req_packages}'.split(), check=True)
|
| 125 |
+
"""
|
| 126 |
+
result = session.run(pip_code)
|
| 127 |
+
verbose_logger.debug(
|
| 128 |
+
"SkillsSandboxExecutor: Installed requirements"
|
| 129 |
+
)
|
| 130 |
+
|
| 131 |
+
# 3. Execute the code
|
| 132 |
+
# Wrap code to run from /sandbox directory
|
| 133 |
+
wrapped_code = f"""
|
| 134 |
+
import os
|
| 135 |
+
os.chdir('/sandbox')
|
| 136 |
+
import sys
|
| 137 |
+
sys.path.insert(0, '/sandbox')
|
| 138 |
+
|
| 139 |
+
{code}
|
| 140 |
+
"""
|
| 141 |
+
result = session.run(wrapped_code)
|
| 142 |
+
|
| 143 |
+
success = result.exit_code == 0
|
| 144 |
+
output = result.stdout or ""
|
| 145 |
+
error = result.stderr or ""
|
| 146 |
+
|
| 147 |
+
if success:
|
| 148 |
+
verbose_logger.debug(
|
| 149 |
+
"SkillsSandboxExecutor: Code execution succeeded"
|
| 150 |
+
)
|
| 151 |
+
else:
|
| 152 |
+
verbose_logger.debug(
|
| 153 |
+
f"SkillsSandboxExecutor: Code execution failed with exit code {result.exit_code}"
|
| 154 |
+
)
|
| 155 |
+
verbose_logger.debug(
|
| 156 |
+
f"SkillsSandboxExecutor: stderr: {error[:500] if error else 'No stderr'}"
|
| 157 |
+
)
|
| 158 |
+
verbose_logger.debug(
|
| 159 |
+
f"SkillsSandboxExecutor: stdout: {output[:500] if output else 'No stdout'}"
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
# 4. Collect generated files
|
| 163 |
+
generated_files = self._collect_generated_files(session, skill_files)
|
| 164 |
+
|
| 165 |
+
return {
|
| 166 |
+
"success": success,
|
| 167 |
+
"output": output,
|
| 168 |
+
"error": error,
|
| 169 |
+
"files": generated_files,
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
except Exception as e:
|
| 173 |
+
verbose_logger.error(f"SkillsSandboxExecutor: Execution failed: {e}")
|
| 174 |
+
return {
|
| 175 |
+
"success": False,
|
| 176 |
+
"output": "",
|
| 177 |
+
"error": str(e),
|
| 178 |
+
"files": [],
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
def _collect_generated_files(
|
| 182 |
+
self,
|
| 183 |
+
session: Any,
|
| 184 |
+
original_files: Dict[str, bytes],
|
| 185 |
+
) -> List[Dict[str, Any]]:
|
| 186 |
+
"""
|
| 187 |
+
Collect files generated during execution.
|
| 188 |
+
|
| 189 |
+
Looks for new files in /sandbox that weren't in the original skill files.
|
| 190 |
+
Focuses on common output types: GIF, PNG, JPG, PDF, CSV, etc.
|
| 191 |
+
|
| 192 |
+
Args:
|
| 193 |
+
session: The sandbox session
|
| 194 |
+
original_files: Original skill files (to exclude)
|
| 195 |
+
|
| 196 |
+
Returns:
|
| 197 |
+
List of generated files with base64 content
|
| 198 |
+
"""
|
| 199 |
+
generated_files: List[Dict[str, Any]] = []
|
| 200 |
+
|
| 201 |
+
try:
|
| 202 |
+
import tempfile
|
| 203 |
+
|
| 204 |
+
# List files in /sandbox using Python code
|
| 205 |
+
list_code = """
|
| 206 |
+
import os
|
| 207 |
+
import json
|
| 208 |
+
files = []
|
| 209 |
+
for root, dirs, filenames in os.walk('/sandbox'):
|
| 210 |
+
for f in filenames:
|
| 211 |
+
if f.endswith(('.gif', '.png', '.jpg', '.jpeg', '.pdf', '.csv', '.json')):
|
| 212 |
+
files.append(os.path.join(root, f))
|
| 213 |
+
print(json.dumps(files))
|
| 214 |
+
"""
|
| 215 |
+
result = session.run(list_code)
|
| 216 |
+
|
| 217 |
+
if result.exit_code == 0 and result.stdout:
|
| 218 |
+
import json
|
| 219 |
+
|
| 220 |
+
try:
|
| 221 |
+
filepaths = json.loads(result.stdout.strip())
|
| 222 |
+
except json.JSONDecodeError:
|
| 223 |
+
filepaths = []
|
| 224 |
+
|
| 225 |
+
for filepath in filepaths:
|
| 226 |
+
if not filepath:
|
| 227 |
+
continue
|
| 228 |
+
|
| 229 |
+
# Get relative path
|
| 230 |
+
rel_path = filepath.replace("/sandbox/", "")
|
| 231 |
+
|
| 232 |
+
# Skip if it was an original file
|
| 233 |
+
if rel_path in original_files:
|
| 234 |
+
continue
|
| 235 |
+
|
| 236 |
+
# Copy file from sandbox using copy_from_runtime
|
| 237 |
+
with tempfile.NamedTemporaryFile(delete=False) as tmp:
|
| 238 |
+
tmp_path = tmp.name
|
| 239 |
+
|
| 240 |
+
try:
|
| 241 |
+
session.copy_from_runtime(filepath, tmp_path)
|
| 242 |
+
|
| 243 |
+
with open(tmp_path, "rb") as f:
|
| 244 |
+
content = f.read()
|
| 245 |
+
|
| 246 |
+
content_b64 = base64.b64encode(content).decode("utf-8")
|
| 247 |
+
generated_files.append(
|
| 248 |
+
{
|
| 249 |
+
"name": os.path.basename(filepath),
|
| 250 |
+
"path": rel_path,
|
| 251 |
+
"content_base64": content_b64,
|
| 252 |
+
"mime_type": self._get_mime_type(filepath),
|
| 253 |
+
}
|
| 254 |
+
)
|
| 255 |
+
|
| 256 |
+
verbose_logger.debug(
|
| 257 |
+
f"SkillsSandboxExecutor: Collected generated file: {rel_path}"
|
| 258 |
+
)
|
| 259 |
+
except Exception as e:
|
| 260 |
+
verbose_logger.warning(
|
| 261 |
+
f"SkillsSandboxExecutor: Error copying file {filepath}: {e}"
|
| 262 |
+
)
|
| 263 |
+
finally:
|
| 264 |
+
if os.path.exists(tmp_path):
|
| 265 |
+
os.unlink(tmp_path)
|
| 266 |
+
|
| 267 |
+
except Exception as e:
|
| 268 |
+
verbose_logger.warning(
|
| 269 |
+
f"SkillsSandboxExecutor: Error collecting generated files: {e}"
|
| 270 |
+
)
|
| 271 |
+
|
| 272 |
+
return generated_files
|
| 273 |
+
|
| 274 |
+
def _get_mime_type(self, filename: str) -> str:
|
| 275 |
+
"""Get MIME type for a file based on extension."""
|
| 276 |
+
ext = filename.lower().split(".")[-1]
|
| 277 |
+
return {
|
| 278 |
+
"gif": "image/gif",
|
| 279 |
+
"png": "image/png",
|
| 280 |
+
"jpg": "image/jpeg",
|
| 281 |
+
"jpeg": "image/jpeg",
|
| 282 |
+
"pdf": "application/pdf",
|
| 283 |
+
"csv": "text/csv",
|
| 284 |
+
"json": "application/json",
|
| 285 |
+
"txt": "text/plain",
|
| 286 |
+
}.get(ext, "application/octet-stream")
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/litellm_proxy/skills/transformation.py
ADDED
|
@@ -0,0 +1,345 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation handler for LiteLLM database-backed skills.
|
| 3 |
+
|
| 4 |
+
This module provides the SDK-level transformation layer that converts
|
| 5 |
+
API requests to database operations via LiteLLMSkillsHandler.
|
| 6 |
+
|
| 7 |
+
Pattern follows litellm/llms/litellm_proxy/responses/transformation.py
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Union
|
| 11 |
+
|
| 12 |
+
from litellm.types.llms.anthropic_skills import (
|
| 13 |
+
DeleteSkillResponse,
|
| 14 |
+
ListSkillsResponse,
|
| 15 |
+
Skill,
|
| 16 |
+
)
|
| 17 |
+
from litellm.types.utils import LlmProviders
|
| 18 |
+
|
| 19 |
+
if TYPE_CHECKING:
|
| 20 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class LiteLLMSkillsTransformationHandler:
|
| 24 |
+
"""
|
| 25 |
+
Transformation handler for skills API requests to LiteLLM database operations.
|
| 26 |
+
|
| 27 |
+
This is used when custom_llm_provider="litellm_proxy" to store/retrieve skills
|
| 28 |
+
from the LiteLLM proxy database instead of calling an external API.
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
@property
|
| 32 |
+
def custom_llm_provider(self) -> str:
|
| 33 |
+
"""Return the provider name for logging."""
|
| 34 |
+
return LlmProviders.LITELLM_PROXY.value
|
| 35 |
+
|
| 36 |
+
def create_skill_handler(
|
| 37 |
+
self,
|
| 38 |
+
display_title: Optional[str] = None,
|
| 39 |
+
description: Optional[str] = None,
|
| 40 |
+
instructions: Optional[str] = None,
|
| 41 |
+
files: Optional[List[Any]] = None,
|
| 42 |
+
file_content: Optional[bytes] = None,
|
| 43 |
+
file_name: Optional[str] = None,
|
| 44 |
+
file_type: Optional[str] = None,
|
| 45 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 46 |
+
user_id: Optional[str] = None,
|
| 47 |
+
_is_async: bool = False,
|
| 48 |
+
logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
| 49 |
+
litellm_call_id: Optional[str] = None,
|
| 50 |
+
**kwargs,
|
| 51 |
+
) -> Union[Skill, Coroutine[Any, Any, Skill]]:
|
| 52 |
+
"""
|
| 53 |
+
Create a skill in LiteLLM database.
|
| 54 |
+
|
| 55 |
+
Args:
|
| 56 |
+
display_title: Display title for the skill
|
| 57 |
+
description: Description of the skill
|
| 58 |
+
instructions: Instructions/prompt for the skill
|
| 59 |
+
files: Files to upload - list of tuples (filename, content, content_type)
|
| 60 |
+
file_content: Binary content of skill files (alternative to files)
|
| 61 |
+
file_name: Original filename (alternative to files)
|
| 62 |
+
file_type: MIME type (alternative to files)
|
| 63 |
+
metadata: Additional metadata
|
| 64 |
+
user_id: User ID for tracking
|
| 65 |
+
_is_async: Whether to return a coroutine
|
| 66 |
+
|
| 67 |
+
Returns:
|
| 68 |
+
Skill object or coroutine that returns Skill
|
| 69 |
+
"""
|
| 70 |
+
# Pre-call logging
|
| 71 |
+
if logging_obj:
|
| 72 |
+
logging_obj.update_from_kwargs(
|
| 73 |
+
kwargs=kwargs,
|
| 74 |
+
model=None,
|
| 75 |
+
optional_params={"display_title": display_title},
|
| 76 |
+
litellm_params={"litellm_call_id": litellm_call_id},
|
| 77 |
+
custom_llm_provider=self.custom_llm_provider,
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
# Extract file content from files parameter if provided
|
| 81 |
+
# files is a list of tuples: [(filename, content, content_type), ...]
|
| 82 |
+
if files and not file_content:
|
| 83 |
+
if isinstance(files, list) and len(files) > 0:
|
| 84 |
+
first_file = files[0]
|
| 85 |
+
if isinstance(first_file, tuple) and len(first_file) >= 2:
|
| 86 |
+
file_name = first_file[0]
|
| 87 |
+
file_content = first_file[1]
|
| 88 |
+
file_type = (
|
| 89 |
+
first_file[2] if len(first_file) > 2 else "application/zip"
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
if _is_async:
|
| 93 |
+
return self._async_create_skill(
|
| 94 |
+
display_title=display_title,
|
| 95 |
+
description=description,
|
| 96 |
+
instructions=instructions,
|
| 97 |
+
file_content=file_content,
|
| 98 |
+
file_name=file_name,
|
| 99 |
+
file_type=file_type,
|
| 100 |
+
metadata=metadata,
|
| 101 |
+
user_id=user_id,
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
import asyncio
|
| 105 |
+
|
| 106 |
+
return asyncio.get_event_loop().run_until_complete(
|
| 107 |
+
self._async_create_skill(
|
| 108 |
+
display_title=display_title,
|
| 109 |
+
description=description,
|
| 110 |
+
instructions=instructions,
|
| 111 |
+
file_content=file_content,
|
| 112 |
+
file_name=file_name,
|
| 113 |
+
file_type=file_type,
|
| 114 |
+
metadata=metadata,
|
| 115 |
+
user_id=user_id,
|
| 116 |
+
)
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
async def _async_create_skill(
|
| 120 |
+
self,
|
| 121 |
+
display_title: Optional[str] = None,
|
| 122 |
+
description: Optional[str] = None,
|
| 123 |
+
instructions: Optional[str] = None,
|
| 124 |
+
file_content: Optional[bytes] = None,
|
| 125 |
+
file_name: Optional[str] = None,
|
| 126 |
+
file_type: Optional[str] = None,
|
| 127 |
+
metadata: Optional[Dict[str, Any]] = None,
|
| 128 |
+
user_id: Optional[str] = None,
|
| 129 |
+
) -> Skill:
|
| 130 |
+
"""Async implementation of create_skill."""
|
| 131 |
+
# Lazy import to avoid SDK dependency on proxy
|
| 132 |
+
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
|
| 133 |
+
from litellm.proxy._types import NewSkillRequest
|
| 134 |
+
|
| 135 |
+
skill_request = NewSkillRequest(
|
| 136 |
+
display_title=display_title,
|
| 137 |
+
description=description,
|
| 138 |
+
instructions=instructions,
|
| 139 |
+
file_content=file_content,
|
| 140 |
+
file_name=file_name,
|
| 141 |
+
file_type=file_type,
|
| 142 |
+
metadata=metadata,
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
db_skill = await LiteLLMSkillsHandler.create_skill(
|
| 146 |
+
data=skill_request,
|
| 147 |
+
user_id=user_id,
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
return self._db_skill_to_response(db_skill)
|
| 151 |
+
|
| 152 |
+
def list_skills_handler(
|
| 153 |
+
self,
|
| 154 |
+
limit: int = 20,
|
| 155 |
+
offset: int = 0,
|
| 156 |
+
_is_async: bool = False,
|
| 157 |
+
logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
| 158 |
+
litellm_call_id: Optional[str] = None,
|
| 159 |
+
**kwargs,
|
| 160 |
+
) -> Union[ListSkillsResponse, Coroutine[Any, Any, ListSkillsResponse]]:
|
| 161 |
+
"""
|
| 162 |
+
List skills from LiteLLM database.
|
| 163 |
+
|
| 164 |
+
Args:
|
| 165 |
+
limit: Maximum number of skills to return
|
| 166 |
+
offset: Number of skills to skip
|
| 167 |
+
_is_async: Whether to return a coroutine
|
| 168 |
+
logging_obj: LiteLLM logging object
|
| 169 |
+
litellm_call_id: Call ID for logging
|
| 170 |
+
|
| 171 |
+
Returns:
|
| 172 |
+
ListSkillsResponse or coroutine that returns ListSkillsResponse
|
| 173 |
+
"""
|
| 174 |
+
# Pre-call logging
|
| 175 |
+
if logging_obj:
|
| 176 |
+
logging_obj.update_from_kwargs(
|
| 177 |
+
kwargs=kwargs,
|
| 178 |
+
model=None,
|
| 179 |
+
optional_params={"limit": limit, "offset": offset},
|
| 180 |
+
litellm_params={"litellm_call_id": litellm_call_id},
|
| 181 |
+
custom_llm_provider=self.custom_llm_provider,
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
if _is_async:
|
| 185 |
+
return self._async_list_skills(limit=limit, offset=offset)
|
| 186 |
+
|
| 187 |
+
import asyncio
|
| 188 |
+
|
| 189 |
+
return asyncio.get_event_loop().run_until_complete(
|
| 190 |
+
self._async_list_skills(limit=limit, offset=offset)
|
| 191 |
+
)
|
| 192 |
+
|
| 193 |
+
async def _async_list_skills(
|
| 194 |
+
self,
|
| 195 |
+
limit: int = 20,
|
| 196 |
+
offset: int = 0,
|
| 197 |
+
) -> ListSkillsResponse:
|
| 198 |
+
"""Async implementation of list_skills."""
|
| 199 |
+
# Lazy import to avoid SDK dependency on proxy
|
| 200 |
+
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
|
| 201 |
+
|
| 202 |
+
db_skills = await LiteLLMSkillsHandler.list_skills(
|
| 203 |
+
limit=limit,
|
| 204 |
+
offset=offset,
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
skills = [self._db_skill_to_response(s) for s in db_skills]
|
| 208 |
+
return ListSkillsResponse(
|
| 209 |
+
data=skills,
|
| 210 |
+
has_more=len(skills) >= limit,
|
| 211 |
+
next_page=None,
|
| 212 |
+
)
|
| 213 |
+
|
| 214 |
+
def get_skill_handler(
|
| 215 |
+
self,
|
| 216 |
+
skill_id: str,
|
| 217 |
+
_is_async: bool = False,
|
| 218 |
+
logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
| 219 |
+
litellm_call_id: Optional[str] = None,
|
| 220 |
+
**kwargs,
|
| 221 |
+
) -> Union[Skill, Coroutine[Any, Any, Skill]]:
|
| 222 |
+
"""
|
| 223 |
+
Get a skill from LiteLLM database.
|
| 224 |
+
|
| 225 |
+
Args:
|
| 226 |
+
skill_id: The skill ID to retrieve
|
| 227 |
+
_is_async: Whether to return a coroutine
|
| 228 |
+
logging_obj: LiteLLM logging object
|
| 229 |
+
litellm_call_id: Call ID for logging
|
| 230 |
+
|
| 231 |
+
Returns:
|
| 232 |
+
Skill or coroutine that returns Skill
|
| 233 |
+
"""
|
| 234 |
+
# Pre-call logging
|
| 235 |
+
if logging_obj:
|
| 236 |
+
logging_obj.update_from_kwargs(
|
| 237 |
+
kwargs=kwargs,
|
| 238 |
+
model=None,
|
| 239 |
+
optional_params={"skill_id": skill_id},
|
| 240 |
+
litellm_params={"litellm_call_id": litellm_call_id},
|
| 241 |
+
custom_llm_provider=self.custom_llm_provider,
|
| 242 |
+
)
|
| 243 |
+
|
| 244 |
+
if _is_async:
|
| 245 |
+
return self._async_get_skill(skill_id=skill_id)
|
| 246 |
+
|
| 247 |
+
import asyncio
|
| 248 |
+
|
| 249 |
+
return asyncio.get_event_loop().run_until_complete(
|
| 250 |
+
self._async_get_skill(skill_id=skill_id)
|
| 251 |
+
)
|
| 252 |
+
|
| 253 |
+
async def _async_get_skill(self, skill_id: str) -> Skill:
|
| 254 |
+
"""Async implementation of get_skill."""
|
| 255 |
+
# Lazy import to avoid SDK dependency on proxy
|
| 256 |
+
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
|
| 257 |
+
|
| 258 |
+
db_skill = await LiteLLMSkillsHandler.get_skill(skill_id=skill_id)
|
| 259 |
+
return self._db_skill_to_response(db_skill)
|
| 260 |
+
|
| 261 |
+
def delete_skill_handler(
|
| 262 |
+
self,
|
| 263 |
+
skill_id: str,
|
| 264 |
+
_is_async: bool = False,
|
| 265 |
+
logging_obj: Optional["LiteLLMLoggingObj"] = None,
|
| 266 |
+
litellm_call_id: Optional[str] = None,
|
| 267 |
+
**kwargs,
|
| 268 |
+
) -> Union[DeleteSkillResponse, Coroutine[Any, Any, DeleteSkillResponse]]:
|
| 269 |
+
"""
|
| 270 |
+
Delete a skill from LiteLLM database.
|
| 271 |
+
|
| 272 |
+
Args:
|
| 273 |
+
skill_id: The skill ID to delete
|
| 274 |
+
_is_async: Whether to return a coroutine
|
| 275 |
+
logging_obj: LiteLLM logging object
|
| 276 |
+
litellm_call_id: Call ID for logging
|
| 277 |
+
|
| 278 |
+
Returns:
|
| 279 |
+
DeleteSkillResponse or coroutine that returns DeleteSkillResponse
|
| 280 |
+
"""
|
| 281 |
+
# Pre-call logging
|
| 282 |
+
if logging_obj:
|
| 283 |
+
logging_obj.update_from_kwargs(
|
| 284 |
+
kwargs=kwargs,
|
| 285 |
+
model=None,
|
| 286 |
+
optional_params={"skill_id": skill_id},
|
| 287 |
+
litellm_params={"litellm_call_id": litellm_call_id},
|
| 288 |
+
custom_llm_provider=self.custom_llm_provider,
|
| 289 |
+
)
|
| 290 |
+
|
| 291 |
+
if _is_async:
|
| 292 |
+
return self._async_delete_skill(skill_id=skill_id)
|
| 293 |
+
|
| 294 |
+
import asyncio
|
| 295 |
+
|
| 296 |
+
return asyncio.get_event_loop().run_until_complete(
|
| 297 |
+
self._async_delete_skill(skill_id=skill_id)
|
| 298 |
+
)
|
| 299 |
+
|
| 300 |
+
async def _async_delete_skill(self, skill_id: str) -> DeleteSkillResponse:
|
| 301 |
+
"""Async implementation of delete_skill."""
|
| 302 |
+
# Lazy import to avoid SDK dependency on proxy
|
| 303 |
+
from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler
|
| 304 |
+
|
| 305 |
+
result = await LiteLLMSkillsHandler.delete_skill(skill_id=skill_id)
|
| 306 |
+
return DeleteSkillResponse(
|
| 307 |
+
id=result["id"],
|
| 308 |
+
type=result.get("type", "skill_deleted"),
|
| 309 |
+
)
|
| 310 |
+
|
| 311 |
+
def _db_skill_to_response(self, db_skill: Any) -> Skill:
|
| 312 |
+
"""
|
| 313 |
+
Convert a database skill record to Anthropic-compatible Skill response.
|
| 314 |
+
|
| 315 |
+
Args:
|
| 316 |
+
db_skill: LiteLLM_SkillsTable record
|
| 317 |
+
|
| 318 |
+
Returns:
|
| 319 |
+
Skill object
|
| 320 |
+
"""
|
| 321 |
+
created_at = ""
|
| 322 |
+
updated_at = ""
|
| 323 |
+
|
| 324 |
+
if hasattr(db_skill, "created_at") and db_skill.created_at:
|
| 325 |
+
created_at = (
|
| 326 |
+
db_skill.created_at.isoformat()
|
| 327 |
+
if hasattr(db_skill.created_at, "isoformat")
|
| 328 |
+
else str(db_skill.created_at)
|
| 329 |
+
)
|
| 330 |
+
if hasattr(db_skill, "updated_at") and db_skill.updated_at:
|
| 331 |
+
updated_at = (
|
| 332 |
+
db_skill.updated_at.isoformat()
|
| 333 |
+
if hasattr(db_skill.updated_at, "isoformat")
|
| 334 |
+
else str(db_skill.updated_at)
|
| 335 |
+
)
|
| 336 |
+
|
| 337 |
+
return Skill(
|
| 338 |
+
id=db_skill.skill_id,
|
| 339 |
+
created_at=created_at,
|
| 340 |
+
updated_at=updated_at,
|
| 341 |
+
display_title=db_skill.display_title,
|
| 342 |
+
latest_version=db_skill.latest_version,
|
| 343 |
+
source=db_skill.source or "custom",
|
| 344 |
+
type="skill",
|
| 345 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/llamafile/chat/transformation.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional, Tuple
|
| 2 |
+
|
| 3 |
+
from litellm.secret_managers.main import get_secret_str
|
| 4 |
+
|
| 5 |
+
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class LlamafileChatConfig(OpenAIGPTConfig):
|
| 9 |
+
"""LlamafileChatConfig is used to provide configuration for the LlamaFile's chat API."""
|
| 10 |
+
|
| 11 |
+
@staticmethod
|
| 12 |
+
def _resolve_api_key(api_key: Optional[str] = None) -> str:
|
| 13 |
+
"""Attempt to ensure that the API key is set, preferring the user-provided key
|
| 14 |
+
over the secret manager key (``LLAMAFILE_API_KEY``).
|
| 15 |
+
|
| 16 |
+
If both are None, a fake API key is returned.
|
| 17 |
+
"""
|
| 18 |
+
return (
|
| 19 |
+
api_key or get_secret_str("LLAMAFILE_API_KEY") or "fake-api-key"
|
| 20 |
+
) # llamafile does not require an API key
|
| 21 |
+
|
| 22 |
+
@staticmethod
|
| 23 |
+
def _resolve_api_base(api_base: Optional[str] = None) -> Optional[str]:
|
| 24 |
+
"""Attempt to ensure that the API base is set, preferring the user-provided key
|
| 25 |
+
over the secret manager key (``LLAMAFILE_API_BASE``).
|
| 26 |
+
|
| 27 |
+
If both are None, a default Llamafile server URL is returned.
|
| 28 |
+
See: https://github.com/Mozilla-Ocho/llamafile/blob/bd1bbe9aabb1ee12dbdcafa8936db443c571eb9d/README.md#L61
|
| 29 |
+
"""
|
| 30 |
+
return api_base or get_secret_str("LLAMAFILE_API_BASE") or "http://127.0.0.1:8080/v1" # type: ignore
|
| 31 |
+
|
| 32 |
+
def _get_openai_compatible_provider_info(
|
| 33 |
+
self, api_base: Optional[str], api_key: Optional[str]
|
| 34 |
+
) -> Tuple[Optional[str], Optional[str]]:
|
| 35 |
+
"""Attempts to ensure that the API base and key are set, preferring user-provided values,
|
| 36 |
+
before falling back to secret manager values (``LLAMAFILE_API_BASE`` and ``LLAMAFILE_API_KEY``
|
| 37 |
+
respectively).
|
| 38 |
+
|
| 39 |
+
If an API key cannot be resolved via either method, a fake key is returned. Llamafile
|
| 40 |
+
does not require an API key, but the underlying OpenAI library may expect one anyway.
|
| 41 |
+
"""
|
| 42 |
+
api_base = LlamafileChatConfig._resolve_api_base(api_base)
|
| 43 |
+
dynamic_api_key = LlamafileChatConfig._resolve_api_key(api_key)
|
| 44 |
+
|
| 45 |
+
return api_base, dynamic_api_key
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/lm_studio/chat/transformation.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Translate from OpenAI's `/v1/chat/completions` to LM Studio's `/chat/completions`
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from typing import Optional, Tuple
|
| 6 |
+
|
| 7 |
+
from litellm.secret_managers.main import get_secret_str
|
| 8 |
+
|
| 9 |
+
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class LMStudioChatConfig(OpenAIGPTConfig):
|
| 13 |
+
def _get_openai_compatible_provider_info(
|
| 14 |
+
self, api_base: Optional[str], api_key: Optional[str]
|
| 15 |
+
) -> Tuple[Optional[str], Optional[str]]:
|
| 16 |
+
api_base = api_base or get_secret_str("LM_STUDIO_API_BASE") # type: ignore
|
| 17 |
+
dynamic_api_key = (
|
| 18 |
+
api_key or get_secret_str("LM_STUDIO_API_KEY") or "fake-api-key"
|
| 19 |
+
) # LM Studio does not require an api key, but OpenAI client requires non-None value
|
| 20 |
+
return api_base, dynamic_api_key
|
| 21 |
+
|
| 22 |
+
def map_openai_params(
|
| 23 |
+
self,
|
| 24 |
+
non_default_params: dict,
|
| 25 |
+
optional_params: dict,
|
| 26 |
+
model: str,
|
| 27 |
+
drop_params: bool,
|
| 28 |
+
) -> dict:
|
| 29 |
+
for param, value in list(non_default_params.items()):
|
| 30 |
+
if param == "response_format" and isinstance(value, dict):
|
| 31 |
+
if value.get("type") == "json_schema":
|
| 32 |
+
if "json_schema" not in value and "schema" in value:
|
| 33 |
+
optional_params["response_format"] = {
|
| 34 |
+
"type": "json_schema",
|
| 35 |
+
"json_schema": {"schema": value.get("schema")},
|
| 36 |
+
}
|
| 37 |
+
else:
|
| 38 |
+
optional_params["response_format"] = value
|
| 39 |
+
non_default_params.pop(param, None)
|
| 40 |
+
elif value.get("type") == "json_object":
|
| 41 |
+
optional_params["response_format"] = value
|
| 42 |
+
non_default_params.pop(param, None)
|
| 43 |
+
|
| 44 |
+
return super().map_openai_params(
|
| 45 |
+
non_default_params=non_default_params,
|
| 46 |
+
optional_params=optional_params,
|
| 47 |
+
model=model,
|
| 48 |
+
drop_params=drop_params,
|
| 49 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/lm_studio/embed/transformation.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation logic from OpenAI /v1/embeddings format to LM Studio's `/v1/embeddings` format.
|
| 3 |
+
|
| 4 |
+
Why separate file? Make it easy to see how transformation works
|
| 5 |
+
|
| 6 |
+
Docs - https://lmstudio.ai/docs/basics/server
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import types
|
| 10 |
+
from typing import List
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class LmStudioEmbeddingConfig:
|
| 14 |
+
"""
|
| 15 |
+
Reference: https://lmstudio.ai/docs/basics/server
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
def __init__(
|
| 19 |
+
self,
|
| 20 |
+
) -> None:
|
| 21 |
+
locals_ = locals().copy()
|
| 22 |
+
for key, value in locals_.items():
|
| 23 |
+
if key != "self" and value is not None:
|
| 24 |
+
setattr(self.__class__, key, value)
|
| 25 |
+
|
| 26 |
+
@classmethod
|
| 27 |
+
def get_config(cls):
|
| 28 |
+
return {
|
| 29 |
+
k: v
|
| 30 |
+
for k, v in cls.__dict__.items()
|
| 31 |
+
if not k.startswith("__")
|
| 32 |
+
and not isinstance(
|
| 33 |
+
v,
|
| 34 |
+
(
|
| 35 |
+
types.FunctionType,
|
| 36 |
+
types.BuiltinFunctionType,
|
| 37 |
+
classmethod,
|
| 38 |
+
staticmethod,
|
| 39 |
+
),
|
| 40 |
+
)
|
| 41 |
+
and v is not None
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
def get_supported_openai_params(self) -> List[str]:
|
| 45 |
+
return []
|
| 46 |
+
|
| 47 |
+
def map_openai_params(
|
| 48 |
+
self, non_default_params: dict, optional_params: dict
|
| 49 |
+
) -> dict:
|
| 50 |
+
return optional_params
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Manus provider implementation
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/files/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Manus Files API implementation
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/files/transformation.py
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Manus Files API implementation.
|
| 3 |
+
|
| 4 |
+
Manus has an OpenAI-compatible Files API with some differences:
|
| 5 |
+
- Uses API_KEY header instead of Authorization: Bearer
|
| 6 |
+
- File upload is a two-step process:
|
| 7 |
+
1. Create file record to get upload URL
|
| 8 |
+
2. Upload file content to the upload URL
|
| 9 |
+
|
| 10 |
+
Reference: https://open.manus.im/docs/openai-compatibility#file-management
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import time
|
| 14 |
+
from typing import Any, Dict, List, Optional, Union
|
| 15 |
+
|
| 16 |
+
import httpx
|
| 17 |
+
from openai.types.file_deleted import FileDeleted
|
| 18 |
+
|
| 19 |
+
import litellm
|
| 20 |
+
from litellm._logging import verbose_logger
|
| 21 |
+
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
|
| 22 |
+
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
| 23 |
+
from litellm.llms.base_llm.files.transformation import (
|
| 24 |
+
BaseFilesConfig,
|
| 25 |
+
LiteLLMLoggingObj,
|
| 26 |
+
)
|
| 27 |
+
from litellm.llms.openai.common_utils import OpenAIError
|
| 28 |
+
from litellm.secret_managers.main import get_secret_str
|
| 29 |
+
from litellm.types.files import TwoStepFileUploadConfig, TwoStepFileUploadRequest
|
| 30 |
+
from litellm.types.llms.openai import (
|
| 31 |
+
CreateFileRequest,
|
| 32 |
+
FileContentRequest,
|
| 33 |
+
HttpxBinaryResponseContent,
|
| 34 |
+
OpenAICreateFileRequestOptionalParams,
|
| 35 |
+
OpenAIFileObject,
|
| 36 |
+
)
|
| 37 |
+
from litellm.types.utils import LlmProviders
|
| 38 |
+
|
| 39 |
+
MANUS_API_BASE = "https://api.manus.im"
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class ManusFilesConfig(BaseFilesConfig):
|
| 43 |
+
"""
|
| 44 |
+
Configuration for Manus Files API.
|
| 45 |
+
|
| 46 |
+
Manus uses:
|
| 47 |
+
- API_KEY header for authentication (not Authorization: Bearer)
|
| 48 |
+
- Two-step file upload process
|
| 49 |
+
- Content-Type: application/json for all requests
|
| 50 |
+
|
| 51 |
+
Reference: https://open.manus.im/docs/openai-compatibility#file-management
|
| 52 |
+
"""
|
| 53 |
+
|
| 54 |
+
def __init__(self):
|
| 55 |
+
pass
|
| 56 |
+
|
| 57 |
+
@property
|
| 58 |
+
def custom_llm_provider(self) -> LlmProviders:
|
| 59 |
+
return LlmProviders.MANUS
|
| 60 |
+
|
| 61 |
+
def validate_environment(
|
| 62 |
+
self,
|
| 63 |
+
headers: dict,
|
| 64 |
+
model: str,
|
| 65 |
+
messages: list,
|
| 66 |
+
optional_params: dict,
|
| 67 |
+
litellm_params: dict,
|
| 68 |
+
api_key: Optional[str] = None,
|
| 69 |
+
api_base: Optional[str] = None,
|
| 70 |
+
) -> dict:
|
| 71 |
+
"""
|
| 72 |
+
Validate environment and set up headers for Manus API.
|
| 73 |
+
|
| 74 |
+
Manus uses API_KEY header instead of Authorization: Bearer.
|
| 75 |
+
For file uploads, don't set Content-Type - httpx will set it for multipart.
|
| 76 |
+
"""
|
| 77 |
+
api_key = api_key or litellm.api_key or get_secret_str("MANUS_API_KEY")
|
| 78 |
+
|
| 79 |
+
if not api_key:
|
| 80 |
+
raise ValueError(
|
| 81 |
+
"Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter."
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
# Manus uses API_KEY header, not Authorization: Bearer
|
| 85 |
+
# Manus requires Content-Type: application/json for all requests (even GET)
|
| 86 |
+
headers.update(
|
| 87 |
+
{
|
| 88 |
+
"API_KEY": api_key,
|
| 89 |
+
"Content-Type": "application/json",
|
| 90 |
+
}
|
| 91 |
+
)
|
| 92 |
+
return headers
|
| 93 |
+
|
| 94 |
+
def get_supported_openai_params(
|
| 95 |
+
self, model: str
|
| 96 |
+
) -> List[OpenAICreateFileRequestOptionalParams]:
|
| 97 |
+
"""
|
| 98 |
+
Return supported OpenAI file creation parameters for Manus.
|
| 99 |
+
Manus supports the standard 'purpose' parameter.
|
| 100 |
+
"""
|
| 101 |
+
return ["purpose"]
|
| 102 |
+
|
| 103 |
+
def map_openai_params(
|
| 104 |
+
self,
|
| 105 |
+
non_default_params: dict,
|
| 106 |
+
optional_params: dict,
|
| 107 |
+
model: str,
|
| 108 |
+
drop_params: bool,
|
| 109 |
+
) -> dict:
|
| 110 |
+
"""
|
| 111 |
+
Map OpenAI parameters to Manus-specific parameters.
|
| 112 |
+
Manus is OpenAI-compatible, so no special mapping needed.
|
| 113 |
+
"""
|
| 114 |
+
return optional_params
|
| 115 |
+
|
| 116 |
+
def get_complete_url(
|
| 117 |
+
self,
|
| 118 |
+
api_base: Optional[str],
|
| 119 |
+
api_key: Optional[str],
|
| 120 |
+
model: str,
|
| 121 |
+
optional_params: dict,
|
| 122 |
+
litellm_params: dict,
|
| 123 |
+
stream: Optional[bool] = None,
|
| 124 |
+
) -> str:
|
| 125 |
+
"""
|
| 126 |
+
Get the complete URL for Manus Files API endpoint.
|
| 127 |
+
|
| 128 |
+
Returns:
|
| 129 |
+
str: The full URL for the Manus /v1/files endpoint
|
| 130 |
+
"""
|
| 131 |
+
api_base = (
|
| 132 |
+
api_base
|
| 133 |
+
or litellm.api_base
|
| 134 |
+
or get_secret_str("MANUS_API_BASE")
|
| 135 |
+
or MANUS_API_BASE
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
# Remove trailing slashes
|
| 139 |
+
api_base = api_base.rstrip("/")
|
| 140 |
+
|
| 141 |
+
# Manus API uses /v1/files endpoint
|
| 142 |
+
if api_base.endswith("/v1"):
|
| 143 |
+
return f"{api_base}/files"
|
| 144 |
+
return f"{api_base}/v1/files"
|
| 145 |
+
|
| 146 |
+
def get_error_class(
|
| 147 |
+
self,
|
| 148 |
+
error_message: str,
|
| 149 |
+
status_code: int,
|
| 150 |
+
headers: Union[dict, httpx.Headers],
|
| 151 |
+
) -> BaseLLMException:
|
| 152 |
+
"""
|
| 153 |
+
Return the appropriate error class for Manus API errors.
|
| 154 |
+
Uses OpenAIError since Manus is OpenAI-compatible.
|
| 155 |
+
"""
|
| 156 |
+
return OpenAIError(
|
| 157 |
+
status_code=status_code,
|
| 158 |
+
message=error_message,
|
| 159 |
+
headers=headers,
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
def transform_create_file_request(
|
| 163 |
+
self,
|
| 164 |
+
model: str,
|
| 165 |
+
create_file_data: CreateFileRequest,
|
| 166 |
+
optional_params: dict,
|
| 167 |
+
litellm_params: dict,
|
| 168 |
+
) -> TwoStepFileUploadConfig:
|
| 169 |
+
"""
|
| 170 |
+
Transform OpenAI-style file creation request into Manus's two-step format.
|
| 171 |
+
|
| 172 |
+
Manus API spec (https://open.manus.im/docs/openai-compatibility#file-management):
|
| 173 |
+
1. POST /v1/files with JSON {"filename": "..."} → returns {"id": "...", "upload_url": "..."}
|
| 174 |
+
2. PUT to upload_url with raw file content
|
| 175 |
+
"""
|
| 176 |
+
# Extract file data
|
| 177 |
+
file_data = create_file_data.get("file")
|
| 178 |
+
if file_data is None:
|
| 179 |
+
raise ValueError("File data is required")
|
| 180 |
+
|
| 181 |
+
extracted_data = extract_file_data(file_data)
|
| 182 |
+
filename = extracted_data["filename"] or f"file_{int(time.time())}"
|
| 183 |
+
content = extracted_data["content"]
|
| 184 |
+
|
| 185 |
+
# Get API base URL
|
| 186 |
+
api_base = self.get_complete_url(
|
| 187 |
+
api_base=litellm_params.get("api_base"),
|
| 188 |
+
api_key=litellm_params.get("api_key"),
|
| 189 |
+
model=model,
|
| 190 |
+
optional_params=optional_params,
|
| 191 |
+
litellm_params=litellm_params,
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
# Get API key
|
| 195 |
+
api_key = (
|
| 196 |
+
litellm_params.get("api_key")
|
| 197 |
+
or litellm.api_key
|
| 198 |
+
or get_secret_str("MANUS_API_KEY")
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
if not api_key:
|
| 202 |
+
raise ValueError(
|
| 203 |
+
"Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter."
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
# Build typed two-step upload config
|
| 207 |
+
return TwoStepFileUploadConfig(
|
| 208 |
+
initial_request=TwoStepFileUploadRequest(
|
| 209 |
+
method="POST",
|
| 210 |
+
url=api_base,
|
| 211 |
+
headers={
|
| 212 |
+
"API_KEY": api_key,
|
| 213 |
+
"Content-Type": "application/json",
|
| 214 |
+
},
|
| 215 |
+
data={"filename": filename},
|
| 216 |
+
),
|
| 217 |
+
upload_request=TwoStepFileUploadRequest(
|
| 218 |
+
method="PUT",
|
| 219 |
+
url="", # Will be populated from initial_request response
|
| 220 |
+
headers={},
|
| 221 |
+
data=content,
|
| 222 |
+
),
|
| 223 |
+
upload_url_location="body",
|
| 224 |
+
upload_url_key="upload_url",
|
| 225 |
+
)
|
| 226 |
+
|
| 227 |
+
def transform_create_file_response(
|
| 228 |
+
self,
|
| 229 |
+
model: Optional[str],
|
| 230 |
+
raw_response: httpx.Response,
|
| 231 |
+
logging_obj: LiteLLMLoggingObj,
|
| 232 |
+
litellm_params: dict,
|
| 233 |
+
) -> OpenAIFileObject:
|
| 234 |
+
"""
|
| 235 |
+
Transform Manus's file upload response into OpenAI-style FileObject.
|
| 236 |
+
|
| 237 |
+
For two-step uploads, the handler stores the initial response in litellm_params.
|
| 238 |
+
We need to return the file object from the initial POST, not the final PUT.
|
| 239 |
+
|
| 240 |
+
Manus initial response format:
|
| 241 |
+
{
|
| 242 |
+
"id": "file-abc123xyz",
|
| 243 |
+
"object": "file",
|
| 244 |
+
"filename": "document.pdf",
|
| 245 |
+
"status": "pending",
|
| 246 |
+
"upload_url": "https://...",
|
| 247 |
+
"upload_expires_at": "...",
|
| 248 |
+
"created_at": "..."
|
| 249 |
+
}
|
| 250 |
+
"""
|
| 251 |
+
try:
|
| 252 |
+
# For two-step uploads, get the initial response from litellm_params
|
| 253 |
+
initial_response_data = litellm_params.get("initial_file_response")
|
| 254 |
+
if initial_response_data:
|
| 255 |
+
response_json = initial_response_data
|
| 256 |
+
else:
|
| 257 |
+
# Log raw response for debugging
|
| 258 |
+
verbose_logger.debug(f"Manus raw response text: {raw_response.text}")
|
| 259 |
+
response_json = raw_response.json()
|
| 260 |
+
|
| 261 |
+
verbose_logger.debug(f"Manus file response: {response_json}")
|
| 262 |
+
|
| 263 |
+
# Parse created_at timestamp
|
| 264 |
+
created_at_str = response_json.get("created_at", "")
|
| 265 |
+
if created_at_str:
|
| 266 |
+
try:
|
| 267 |
+
# Try parsing ISO format
|
| 268 |
+
created_at = int(
|
| 269 |
+
time.mktime(
|
| 270 |
+
time.strptime(
|
| 271 |
+
created_at_str.replace("Z", "+00:00")[:19],
|
| 272 |
+
"%Y-%m-%dT%H:%M:%S",
|
| 273 |
+
)
|
| 274 |
+
)
|
| 275 |
+
)
|
| 276 |
+
except (ValueError, TypeError):
|
| 277 |
+
created_at = int(time.time())
|
| 278 |
+
else:
|
| 279 |
+
created_at = int(time.time())
|
| 280 |
+
|
| 281 |
+
return OpenAIFileObject(
|
| 282 |
+
id=response_json.get("id", ""),
|
| 283 |
+
bytes=response_json.get("bytes", 0),
|
| 284 |
+
created_at=created_at,
|
| 285 |
+
filename=response_json.get("filename", ""),
|
| 286 |
+
object="file",
|
| 287 |
+
purpose=response_json.get("purpose", "assistants"),
|
| 288 |
+
status="uploaded", # After successful upload, status is uploaded
|
| 289 |
+
status_details=response_json.get("status_details"),
|
| 290 |
+
)
|
| 291 |
+
except Exception as e:
|
| 292 |
+
verbose_logger.exception(f"Error parsing Manus file response: {str(e)}")
|
| 293 |
+
raise ValueError(f"Error parsing Manus file response: {str(e)}")
|
| 294 |
+
|
| 295 |
+
def transform_retrieve_file_request(
|
| 296 |
+
self,
|
| 297 |
+
file_id: str,
|
| 298 |
+
optional_params: dict,
|
| 299 |
+
litellm_params: dict,
|
| 300 |
+
) -> tuple[str, dict]:
|
| 301 |
+
"""Get URL and params for retrieving a file."""
|
| 302 |
+
api_base = self.get_complete_url(
|
| 303 |
+
api_base=litellm_params.get("api_base"),
|
| 304 |
+
api_key=litellm_params.get("api_key"),
|
| 305 |
+
model="",
|
| 306 |
+
optional_params=optional_params,
|
| 307 |
+
litellm_params=litellm_params,
|
| 308 |
+
)
|
| 309 |
+
return f"{api_base}/{file_id}", {}
|
| 310 |
+
|
| 311 |
+
def transform_retrieve_file_response(
|
| 312 |
+
self,
|
| 313 |
+
raw_response: httpx.Response,
|
| 314 |
+
logging_obj: LiteLLMLoggingObj,
|
| 315 |
+
litellm_params: dict,
|
| 316 |
+
) -> OpenAIFileObject:
|
| 317 |
+
"""Transform retrieve file response."""
|
| 318 |
+
return self.transform_create_file_response(
|
| 319 |
+
model=None,
|
| 320 |
+
raw_response=raw_response,
|
| 321 |
+
logging_obj=logging_obj,
|
| 322 |
+
litellm_params=litellm_params,
|
| 323 |
+
)
|
| 324 |
+
|
| 325 |
+
def transform_delete_file_request(
|
| 326 |
+
self,
|
| 327 |
+
file_id: str,
|
| 328 |
+
optional_params: dict,
|
| 329 |
+
litellm_params: dict,
|
| 330 |
+
) -> tuple[str, dict]:
|
| 331 |
+
"""Get URL and params for deleting a file."""
|
| 332 |
+
api_base = self.get_complete_url(
|
| 333 |
+
api_base=litellm_params.get("api_base"),
|
| 334 |
+
api_key=litellm_params.get("api_key"),
|
| 335 |
+
model="",
|
| 336 |
+
optional_params=optional_params,
|
| 337 |
+
litellm_params=litellm_params,
|
| 338 |
+
)
|
| 339 |
+
return f"{api_base}/{file_id}", {}
|
| 340 |
+
|
| 341 |
+
def transform_delete_file_response(
|
| 342 |
+
self,
|
| 343 |
+
raw_response: httpx.Response,
|
| 344 |
+
logging_obj: LiteLLMLoggingObj,
|
| 345 |
+
litellm_params: dict,
|
| 346 |
+
) -> FileDeleted:
|
| 347 |
+
"""Transform delete file response."""
|
| 348 |
+
response_json = raw_response.json()
|
| 349 |
+
return FileDeleted(**response_json)
|
| 350 |
+
|
| 351 |
+
def transform_list_files_request(
|
| 352 |
+
self,
|
| 353 |
+
purpose: Optional[str],
|
| 354 |
+
optional_params: dict,
|
| 355 |
+
litellm_params: dict,
|
| 356 |
+
) -> tuple[str, dict]:
|
| 357 |
+
"""Get URL and params for listing files."""
|
| 358 |
+
api_base = self.get_complete_url(
|
| 359 |
+
api_base=litellm_params.get("api_base"),
|
| 360 |
+
api_key=litellm_params.get("api_key"),
|
| 361 |
+
model="",
|
| 362 |
+
optional_params=optional_params,
|
| 363 |
+
litellm_params=litellm_params,
|
| 364 |
+
)
|
| 365 |
+
params = {}
|
| 366 |
+
if purpose:
|
| 367 |
+
params["purpose"] = purpose
|
| 368 |
+
return api_base, params
|
| 369 |
+
|
| 370 |
+
def transform_list_files_response(
|
| 371 |
+
self,
|
| 372 |
+
raw_response: httpx.Response,
|
| 373 |
+
logging_obj: LiteLLMLoggingObj,
|
| 374 |
+
litellm_params: dict,
|
| 375 |
+
) -> List[OpenAIFileObject]:
|
| 376 |
+
"""Transform list files response."""
|
| 377 |
+
response_json = raw_response.json()
|
| 378 |
+
files_data = response_json.get("data", [])
|
| 379 |
+
return [self._parse_file_dict(f) for f in files_data]
|
| 380 |
+
|
| 381 |
+
def _parse_file_dict(self, file_dict: Dict[str, Any]) -> OpenAIFileObject:
|
| 382 |
+
"""Parse a file dict into OpenAIFileObject."""
|
| 383 |
+
created_at_str = file_dict.get("created_at", "")
|
| 384 |
+
if created_at_str:
|
| 385 |
+
try:
|
| 386 |
+
created_at = int(
|
| 387 |
+
time.mktime(
|
| 388 |
+
time.strptime(
|
| 389 |
+
created_at_str.replace("Z", "+00:00")[:19],
|
| 390 |
+
"%Y-%m-%dT%H:%M:%S",
|
| 391 |
+
)
|
| 392 |
+
)
|
| 393 |
+
)
|
| 394 |
+
except (ValueError, TypeError):
|
| 395 |
+
created_at = int(time.time())
|
| 396 |
+
else:
|
| 397 |
+
created_at = int(time.time())
|
| 398 |
+
|
| 399 |
+
return OpenAIFileObject(
|
| 400 |
+
id=file_dict.get("id", ""),
|
| 401 |
+
bytes=file_dict.get("bytes", 0),
|
| 402 |
+
created_at=created_at,
|
| 403 |
+
filename=file_dict.get("filename", ""),
|
| 404 |
+
object="file",
|
| 405 |
+
purpose=file_dict.get("purpose", "assistants"),
|
| 406 |
+
status=file_dict.get("status", "uploaded"),
|
| 407 |
+
status_details=file_dict.get("status_details"),
|
| 408 |
+
)
|
| 409 |
+
|
| 410 |
+
def transform_file_content_request(
|
| 411 |
+
self,
|
| 412 |
+
file_content_request: FileContentRequest,
|
| 413 |
+
optional_params: dict,
|
| 414 |
+
litellm_params: dict,
|
| 415 |
+
) -> tuple[str, dict]:
|
| 416 |
+
"""Get URL and params for retrieving file content."""
|
| 417 |
+
file_id = file_content_request.get("file_id")
|
| 418 |
+
api_base = self.get_complete_url(
|
| 419 |
+
api_base=litellm_params.get("api_base"),
|
| 420 |
+
api_key=litellm_params.get("api_key"),
|
| 421 |
+
model="",
|
| 422 |
+
optional_params=optional_params,
|
| 423 |
+
litellm_params=litellm_params,
|
| 424 |
+
)
|
| 425 |
+
return f"{api_base}/{file_id}/content", {}
|
| 426 |
+
|
| 427 |
+
def transform_file_content_response(
|
| 428 |
+
self,
|
| 429 |
+
raw_response: httpx.Response,
|
| 430 |
+
logging_obj: LiteLLMLoggingObj,
|
| 431 |
+
litellm_params: dict,
|
| 432 |
+
) -> HttpxBinaryResponseContent:
|
| 433 |
+
"""Transform file content response."""
|
| 434 |
+
return HttpxBinaryResponseContent(response=raw_response)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/responses/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
# Manus Responses API implementation
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/manus/responses/transformation.py
ADDED
|
@@ -0,0 +1,353 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import uuid
|
| 2 |
+
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
|
| 3 |
+
|
| 4 |
+
import httpx
|
| 5 |
+
|
| 6 |
+
import litellm
|
| 7 |
+
from litellm._logging import verbose_logger
|
| 8 |
+
from litellm.litellm_core_utils.core_helpers import process_response_headers
|
| 9 |
+
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
| 10 |
+
_safe_convert_created_field,
|
| 11 |
+
)
|
| 12 |
+
from litellm.llms.openai.common_utils import OpenAIError
|
| 13 |
+
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
| 14 |
+
from litellm.secret_managers.main import get_secret_str
|
| 15 |
+
from litellm.types.llms.openai import (
|
| 16 |
+
ResponseAPIUsage,
|
| 17 |
+
ResponseInputParam,
|
| 18 |
+
ResponsesAPIResponse,
|
| 19 |
+
)
|
| 20 |
+
from litellm.types.router import GenericLiteLLMParams
|
| 21 |
+
from litellm.types.utils import LlmProviders
|
| 22 |
+
|
| 23 |
+
if TYPE_CHECKING:
|
| 24 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
| 25 |
+
|
| 26 |
+
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
| 27 |
+
else:
|
| 28 |
+
LiteLLMLoggingObj = Any
|
| 29 |
+
|
| 30 |
+
MANUS_API_BASE = "https://api.manus.im"
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
| 34 |
+
"""
|
| 35 |
+
Configuration for Manus API's Responses API.
|
| 36 |
+
|
| 37 |
+
Manus API is OpenAI-compatible but has some differences:
|
| 38 |
+
- API key passed via `API_KEY` header (not `Authorization: Bearer`)
|
| 39 |
+
- Model format: `manus/{agent_profile}` (e.g., `manus/manus-1.6`)
|
| 40 |
+
- Requires `extra_body` with `task_mode: "agent"` and `agent_profile`
|
| 41 |
+
|
| 42 |
+
Reference: https://open.manus.im/docs/openai-compatibility
|
| 43 |
+
"""
|
| 44 |
+
|
| 45 |
+
@property
|
| 46 |
+
def custom_llm_provider(self) -> LlmProviders:
|
| 47 |
+
return LlmProviders.MANUS
|
| 48 |
+
|
| 49 |
+
def should_fake_stream(
|
| 50 |
+
self,
|
| 51 |
+
model: Optional[str],
|
| 52 |
+
stream: Optional[bool],
|
| 53 |
+
custom_llm_provider: Optional[str] = None,
|
| 54 |
+
) -> bool:
|
| 55 |
+
"""
|
| 56 |
+
Manus API doesn't support real-time streaming.
|
| 57 |
+
It returns a task that runs asynchronously.
|
| 58 |
+
We fake streaming by converting the response into streaming events.
|
| 59 |
+
"""
|
| 60 |
+
return stream is True
|
| 61 |
+
|
| 62 |
+
def _extract_agent_profile(self, model: str) -> str:
|
| 63 |
+
"""
|
| 64 |
+
Extract agent profile from model name.
|
| 65 |
+
|
| 66 |
+
Model format: `manus/{agent_profile}`
|
| 67 |
+
Examples: `manus/manus-1.6`, `manus/manus-1.6-lite`, `manus/manus-1.6-max`
|
| 68 |
+
|
| 69 |
+
Returns:
|
| 70 |
+
str: The agent profile (e.g., "manus-1.6")
|
| 71 |
+
"""
|
| 72 |
+
if "/" in model:
|
| 73 |
+
return model.split("/", 1)[1]
|
| 74 |
+
# If no slash, assume the model name itself is the agent profile
|
| 75 |
+
return model
|
| 76 |
+
|
| 77 |
+
def validate_environment(
|
| 78 |
+
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
|
| 79 |
+
) -> dict:
|
| 80 |
+
"""
|
| 81 |
+
Validate environment and set up headers for Manus API.
|
| 82 |
+
|
| 83 |
+
Manus uses `API_KEY` header instead of `Authorization: Bearer`.
|
| 84 |
+
"""
|
| 85 |
+
litellm_params = litellm_params or GenericLiteLLMParams()
|
| 86 |
+
api_key = (
|
| 87 |
+
litellm_params.api_key or litellm.api_key or get_secret_str("MANUS_API_KEY")
|
| 88 |
+
)
|
| 89 |
+
|
| 90 |
+
if not api_key:
|
| 91 |
+
raise ValueError(
|
| 92 |
+
"Manus API key is required. Set MANUS_API_KEY environment variable or pass api_key parameter."
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
# Manus uses API_KEY header, not Authorization: Bearer
|
| 96 |
+
# Content-Type is required for all requests (including GET)
|
| 97 |
+
headers.update(
|
| 98 |
+
{
|
| 99 |
+
"API_KEY": api_key,
|
| 100 |
+
"Content-Type": "application/json",
|
| 101 |
+
}
|
| 102 |
+
)
|
| 103 |
+
return headers
|
| 104 |
+
|
| 105 |
+
def get_complete_url(
|
| 106 |
+
self,
|
| 107 |
+
api_base: Optional[str],
|
| 108 |
+
litellm_params: dict,
|
| 109 |
+
) -> str:
|
| 110 |
+
"""
|
| 111 |
+
Get the complete URL for Manus Responses API endpoint.
|
| 112 |
+
|
| 113 |
+
Returns:
|
| 114 |
+
str: The full URL for the Manus /v1/responses endpoint
|
| 115 |
+
"""
|
| 116 |
+
api_base = (
|
| 117 |
+
api_base
|
| 118 |
+
or litellm.api_base
|
| 119 |
+
or get_secret_str("MANUS_API_BASE")
|
| 120 |
+
or MANUS_API_BASE
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
# Remove trailing slashes
|
| 124 |
+
api_base = api_base.rstrip("/")
|
| 125 |
+
|
| 126 |
+
# Manus API uses /v1/responses endpoint (OpenAI-compatible)
|
| 127 |
+
if api_base.endswith("/v1"):
|
| 128 |
+
return f"{api_base}/responses"
|
| 129 |
+
return f"{api_base}/v1/responses"
|
| 130 |
+
|
| 131 |
+
def transform_responses_api_request(
|
| 132 |
+
self,
|
| 133 |
+
model: str,
|
| 134 |
+
input: Union[str, ResponseInputParam],
|
| 135 |
+
response_api_optional_request_params: Dict,
|
| 136 |
+
litellm_params: GenericLiteLLMParams,
|
| 137 |
+
headers: dict,
|
| 138 |
+
) -> Dict:
|
| 139 |
+
"""
|
| 140 |
+
Transform the request for Manus API.
|
| 141 |
+
|
| 142 |
+
Manus requires:
|
| 143 |
+
- `task_mode: "agent"` in the request body
|
| 144 |
+
- `agent_profile` extracted from model name in the request body
|
| 145 |
+
"""
|
| 146 |
+
# First, get the base OpenAI request
|
| 147 |
+
base_request = super().transform_responses_api_request(
|
| 148 |
+
model=model,
|
| 149 |
+
input=input,
|
| 150 |
+
response_api_optional_request_params=response_api_optional_request_params,
|
| 151 |
+
litellm_params=litellm_params,
|
| 152 |
+
headers=headers,
|
| 153 |
+
)
|
| 154 |
+
|
| 155 |
+
# Extract agent profile from model name
|
| 156 |
+
agent_profile = self._extract_agent_profile(model=model)
|
| 157 |
+
|
| 158 |
+
# Add Manus-specific parameters directly to the request body
|
| 159 |
+
# These will be sent as part of the request
|
| 160 |
+
base_request["task_mode"] = "agent"
|
| 161 |
+
base_request["agent_profile"] = agent_profile
|
| 162 |
+
|
| 163 |
+
# Merge any existing extra_body into the request
|
| 164 |
+
extra_body = response_api_optional_request_params.get("extra_body", {}) or {}
|
| 165 |
+
if extra_body:
|
| 166 |
+
base_request.update(extra_body)
|
| 167 |
+
|
| 168 |
+
verbose_logger.debug(
|
| 169 |
+
f"Manus: Using agent_profile={agent_profile}, task_mode=agent"
|
| 170 |
+
)
|
| 171 |
+
|
| 172 |
+
return base_request
|
| 173 |
+
|
| 174 |
+
def transform_response_api_response(
|
| 175 |
+
self,
|
| 176 |
+
model: str,
|
| 177 |
+
raw_response: httpx.Response,
|
| 178 |
+
logging_obj: LiteLLMLoggingObj,
|
| 179 |
+
) -> ResponsesAPIResponse:
|
| 180 |
+
"""
|
| 181 |
+
Transform Manus API response to OpenAI-compatible format.
|
| 182 |
+
|
| 183 |
+
Manus uses camelCase (createdAt) instead of snake_case (created_at).
|
| 184 |
+
"""
|
| 185 |
+
try:
|
| 186 |
+
logging_obj.post_call(
|
| 187 |
+
original_response=raw_response.text,
|
| 188 |
+
additional_args={"complete_input_dict": {}},
|
| 189 |
+
)
|
| 190 |
+
raw_response_json = raw_response.json()
|
| 191 |
+
|
| 192 |
+
# Manus uses camelCase "createdAt" instead of snake_case "created_at"
|
| 193 |
+
if (
|
| 194 |
+
"createdAt" in raw_response_json
|
| 195 |
+
and "created_at" not in raw_response_json
|
| 196 |
+
):
|
| 197 |
+
raw_response_json["created_at"] = _safe_convert_created_field(
|
| 198 |
+
raw_response_json["createdAt"]
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
# Ensure created_at is set
|
| 202 |
+
if "created_at" in raw_response_json:
|
| 203 |
+
raw_response_json["created_at"] = _safe_convert_created_field(
|
| 204 |
+
raw_response_json["created_at"]
|
| 205 |
+
)
|
| 206 |
+
except Exception:
|
| 207 |
+
raise OpenAIError(
|
| 208 |
+
message=raw_response.text, status_code=raw_response.status_code
|
| 209 |
+
)
|
| 210 |
+
|
| 211 |
+
raw_response_headers = dict(raw_response.headers)
|
| 212 |
+
processed_headers = process_response_headers(raw_response_headers)
|
| 213 |
+
|
| 214 |
+
# Ensure reasoning is an empty dict if not present, OpenAI SDK does not allow None
|
| 215 |
+
if (
|
| 216 |
+
"reasoning" not in raw_response_json
|
| 217 |
+
or raw_response_json.get("reasoning") is None
|
| 218 |
+
):
|
| 219 |
+
raw_response_json["reasoning"] = {}
|
| 220 |
+
|
| 221 |
+
if "text" not in raw_response_json or raw_response_json.get("text") is None:
|
| 222 |
+
raw_response_json["text"] = {}
|
| 223 |
+
|
| 224 |
+
if "output" not in raw_response_json or raw_response_json.get("output") is None:
|
| 225 |
+
raw_response_json["output"] = []
|
| 226 |
+
|
| 227 |
+
# Ensure usage is present with default values if not provided
|
| 228 |
+
if "usage" not in raw_response_json or raw_response_json.get("usage") is None:
|
| 229 |
+
raw_response_json["usage"] = ResponseAPIUsage(
|
| 230 |
+
input_tokens=0,
|
| 231 |
+
output_tokens=0,
|
| 232 |
+
total_tokens=0,
|
| 233 |
+
)
|
| 234 |
+
|
| 235 |
+
# Ensure id is present - failed responses may not include it
|
| 236 |
+
if "id" not in raw_response_json or raw_response_json.get("id") is None:
|
| 237 |
+
# Generate a placeholder id for failed responses
|
| 238 |
+
# This allows the response object to be created even when the API doesn't return an id
|
| 239 |
+
raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
|
| 240 |
+
|
| 241 |
+
try:
|
| 242 |
+
response = ResponsesAPIResponse(**raw_response_json)
|
| 243 |
+
except Exception:
|
| 244 |
+
verbose_logger.debug(
|
| 245 |
+
f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct"
|
| 246 |
+
)
|
| 247 |
+
response = ResponsesAPIResponse.model_construct(**raw_response_json)
|
| 248 |
+
|
| 249 |
+
# Store processed headers in additional_headers so they get returned to the client
|
| 250 |
+
response._hidden_params["additional_headers"] = processed_headers
|
| 251 |
+
response._hidden_params["headers"] = raw_response_headers
|
| 252 |
+
return response
|
| 253 |
+
|
| 254 |
+
def supports_native_websocket(self) -> bool:
|
| 255 |
+
"""Manus does not support native WebSocket for Responses API"""
|
| 256 |
+
return False
|
| 257 |
+
|
| 258 |
+
def transform_get_response_api_request(
|
| 259 |
+
self,
|
| 260 |
+
response_id: str,
|
| 261 |
+
api_base: str,
|
| 262 |
+
litellm_params: GenericLiteLLMParams,
|
| 263 |
+
headers: dict,
|
| 264 |
+
) -> Tuple[str, Dict]:
|
| 265 |
+
"""
|
| 266 |
+
Transform the get response API request into a URL and data.
|
| 267 |
+
|
| 268 |
+
Manus API follows OpenAI-compatible format:
|
| 269 |
+
- GET /v1/responses/{response_id}
|
| 270 |
+
|
| 271 |
+
Reference: https://open.manus.im/docs/openai-compatibility
|
| 272 |
+
"""
|
| 273 |
+
url = f"{api_base}/{response_id}"
|
| 274 |
+
data: Dict = {}
|
| 275 |
+
return url, data
|
| 276 |
+
|
| 277 |
+
def transform_get_response_api_response(
|
| 278 |
+
self,
|
| 279 |
+
raw_response: httpx.Response,
|
| 280 |
+
logging_obj: LiteLLMLoggingObj,
|
| 281 |
+
) -> ResponsesAPIResponse:
|
| 282 |
+
"""
|
| 283 |
+
Transform Manus API GET response to OpenAI-compatible format.
|
| 284 |
+
|
| 285 |
+
Manus uses camelCase (createdAt) instead of snake_case (created_at).
|
| 286 |
+
Same transformation as transform_response_api_response.
|
| 287 |
+
"""
|
| 288 |
+
try:
|
| 289 |
+
logging_obj.post_call(
|
| 290 |
+
original_response=raw_response.text,
|
| 291 |
+
additional_args={"complete_input_dict": {}},
|
| 292 |
+
)
|
| 293 |
+
raw_response_json = raw_response.json()
|
| 294 |
+
|
| 295 |
+
# Manus uses camelCase "createdAt" instead of snake_case "created_at"
|
| 296 |
+
if (
|
| 297 |
+
"createdAt" in raw_response_json
|
| 298 |
+
and "created_at" not in raw_response_json
|
| 299 |
+
):
|
| 300 |
+
raw_response_json["created_at"] = _safe_convert_created_field(
|
| 301 |
+
raw_response_json["createdAt"]
|
| 302 |
+
)
|
| 303 |
+
|
| 304 |
+
# Ensure created_at is set
|
| 305 |
+
if "created_at" in raw_response_json:
|
| 306 |
+
raw_response_json["created_at"] = _safe_convert_created_field(
|
| 307 |
+
raw_response_json["created_at"]
|
| 308 |
+
)
|
| 309 |
+
except Exception:
|
| 310 |
+
raise OpenAIError(
|
| 311 |
+
message=raw_response.text, status_code=raw_response.status_code
|
| 312 |
+
)
|
| 313 |
+
|
| 314 |
+
raw_response_headers = dict(raw_response.headers)
|
| 315 |
+
processed_headers = process_response_headers(raw_response_headers)
|
| 316 |
+
|
| 317 |
+
# Ensure reasoning, text, output, and usage are present with defaults
|
| 318 |
+
if (
|
| 319 |
+
"reasoning" not in raw_response_json
|
| 320 |
+
or raw_response_json.get("reasoning") is None
|
| 321 |
+
):
|
| 322 |
+
raw_response_json["reasoning"] = {}
|
| 323 |
+
|
| 324 |
+
if "text" not in raw_response_json or raw_response_json.get("text") is None:
|
| 325 |
+
raw_response_json["text"] = {}
|
| 326 |
+
|
| 327 |
+
if "output" not in raw_response_json or raw_response_json.get("output") is None:
|
| 328 |
+
raw_response_json["output"] = []
|
| 329 |
+
|
| 330 |
+
if "usage" not in raw_response_json or raw_response_json.get("usage") is None:
|
| 331 |
+
raw_response_json["usage"] = ResponseAPIUsage(
|
| 332 |
+
input_tokens=0,
|
| 333 |
+
output_tokens=0,
|
| 334 |
+
total_tokens=0,
|
| 335 |
+
)
|
| 336 |
+
|
| 337 |
+
# Ensure id is present - failed responses may not include it
|
| 338 |
+
if "id" not in raw_response_json or raw_response_json.get("id") is None:
|
| 339 |
+
# Generate a placeholder id for failed responses
|
| 340 |
+
raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}"
|
| 341 |
+
|
| 342 |
+
try:
|
| 343 |
+
response = ResponsesAPIResponse(**raw_response_json)
|
| 344 |
+
except Exception:
|
| 345 |
+
verbose_logger.debug(
|
| 346 |
+
f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct"
|
| 347 |
+
)
|
| 348 |
+
response = ResponsesAPIResponse.model_construct(**raw_response_json)
|
| 349 |
+
|
| 350 |
+
# Store processed headers in additional_headers so they get returned to the client
|
| 351 |
+
response._hidden_params["additional_headers"] = processed_headers
|
| 352 |
+
response._hidden_params["headers"] = raw_response_headers
|
| 353 |
+
return response
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/maritalk.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import List, Optional, Union
|
| 2 |
+
|
| 3 |
+
from httpx._models import Headers
|
| 4 |
+
|
| 5 |
+
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
| 6 |
+
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class MaritalkError(BaseLLMException):
|
| 10 |
+
def __init__(
|
| 11 |
+
self,
|
| 12 |
+
status_code: int,
|
| 13 |
+
message: str,
|
| 14 |
+
headers: Optional[Union[dict, Headers]] = None,
|
| 15 |
+
):
|
| 16 |
+
super().__init__(status_code=status_code, message=message, headers=headers)
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class MaritalkConfig(OpenAIGPTConfig):
|
| 20 |
+
def __init__(
|
| 21 |
+
self,
|
| 22 |
+
frequency_penalty: Optional[float] = None,
|
| 23 |
+
presence_penalty: Optional[float] = None,
|
| 24 |
+
top_p: Optional[float] = None,
|
| 25 |
+
top_k: Optional[int] = None,
|
| 26 |
+
temperature: Optional[float] = None,
|
| 27 |
+
max_tokens: Optional[int] = None,
|
| 28 |
+
n: Optional[int] = None,
|
| 29 |
+
stop: Optional[List[str]] = None,
|
| 30 |
+
stream: Optional[bool] = None,
|
| 31 |
+
stream_options: Optional[dict] = None,
|
| 32 |
+
tools: Optional[List[dict]] = None,
|
| 33 |
+
tool_choice: Optional[Union[str, dict]] = None,
|
| 34 |
+
) -> None:
|
| 35 |
+
locals_ = locals().copy()
|
| 36 |
+
for key, value in locals_.items():
|
| 37 |
+
if key != "self" and value is not None:
|
| 38 |
+
setattr(self.__class__, key, value)
|
| 39 |
+
|
| 40 |
+
@classmethod
|
| 41 |
+
def get_config(cls):
|
| 42 |
+
return super().get_config()
|
| 43 |
+
|
| 44 |
+
def get_supported_openai_params(self, model: str) -> List:
|
| 45 |
+
return [
|
| 46 |
+
"frequency_penalty",
|
| 47 |
+
"presence_penalty",
|
| 48 |
+
"top_p",
|
| 49 |
+
"top_k",
|
| 50 |
+
"temperature",
|
| 51 |
+
"max_tokens",
|
| 52 |
+
"n",
|
| 53 |
+
"stop",
|
| 54 |
+
"stream",
|
| 55 |
+
"stream_options",
|
| 56 |
+
"tools",
|
| 57 |
+
"tool_choice",
|
| 58 |
+
]
|
| 59 |
+
|
| 60 |
+
def get_error_class(
|
| 61 |
+
self, error_message: str, status_code: int, headers: Union[dict, Headers]
|
| 62 |
+
) -> BaseLLMException:
|
| 63 |
+
return MaritalkError(
|
| 64 |
+
status_code=status_code, message=error_message, headers=headers
|
| 65 |
+
)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/meta_llama/chat/transformation.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Support for Llama API's `https://api.llama.com/compat/v1` endpoint.
|
| 3 |
+
|
| 4 |
+
Calls done in OpenAI/openai.py as Llama API is openai-compatible.
|
| 5 |
+
|
| 6 |
+
Docs: https://llama.developer.meta.com/docs/features/compatibility/
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
import warnings
|
| 10 |
+
|
| 11 |
+
# Suppress Pydantic serialization warnings for Meta Llama responses
|
| 12 |
+
warnings.filterwarnings("ignore", message="Pydantic serializer warnings")
|
| 13 |
+
|
| 14 |
+
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class LlamaAPIConfig(OpenAIGPTConfig):
|
| 18 |
+
def get_supported_openai_params(self, model: str) -> list:
|
| 19 |
+
"""
|
| 20 |
+
Llama API has limited support for OpenAI parameters
|
| 21 |
+
|
| 22 |
+
function_call, tools, and tool_choice are working
|
| 23 |
+
response_format: only json_schema is working
|
| 24 |
+
"""
|
| 25 |
+
# Function calling and tool choice are now supported on Llama API
|
| 26 |
+
optional_params = super().get_supported_openai_params(model)
|
| 27 |
+
return optional_params
|
| 28 |
+
|
| 29 |
+
def map_openai_params(
|
| 30 |
+
self,
|
| 31 |
+
non_default_params: dict,
|
| 32 |
+
optional_params: dict,
|
| 33 |
+
model: str,
|
| 34 |
+
drop_params: bool,
|
| 35 |
+
) -> dict:
|
| 36 |
+
mapped_openai_params = super().map_openai_params(
|
| 37 |
+
non_default_params, optional_params, model, drop_params
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
# Only json_schema is working for response_format
|
| 41 |
+
if (
|
| 42 |
+
"response_format" in mapped_openai_params
|
| 43 |
+
and mapped_openai_params["response_format"].get("type") != "json_schema"
|
| 44 |
+
):
|
| 45 |
+
mapped_openai_params.pop("response_format")
|
| 46 |
+
return mapped_openai_params
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/milvus/vector_stores/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from litellm.llms.milvus.vector_stores.transformation import MilvusVectorStoreConfig
|
| 2 |
+
|
| 3 |
+
__all__ = ["MilvusVectorStoreConfig"]
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/milvus/vector_stores/transformation.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
| 2 |
+
|
| 3 |
+
import httpx
|
| 4 |
+
|
| 5 |
+
import litellm
|
| 6 |
+
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
|
| 7 |
+
from litellm.secret_managers.main import get_secret_str
|
| 8 |
+
from litellm.types.router import GenericLiteLLMParams
|
| 9 |
+
from litellm.types.vector_stores import (
|
| 10 |
+
BaseVectorStoreAuthCredentials,
|
| 11 |
+
VectorStoreCreateOptionalRequestParams,
|
| 12 |
+
VectorStoreCreateResponse,
|
| 13 |
+
VectorStoreIndexEndpoints,
|
| 14 |
+
VectorStoreResultContent,
|
| 15 |
+
VectorStoreSearchOptionalRequestParams,
|
| 16 |
+
VectorStoreSearchResponse,
|
| 17 |
+
VectorStoreSearchResult,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
if TYPE_CHECKING:
|
| 21 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
| 22 |
+
|
| 23 |
+
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
| 24 |
+
else:
|
| 25 |
+
LiteLLMLoggingObj = Any
|
| 26 |
+
|
| 27 |
+
MILVUS_OPTIONAL_PARAMS = {
|
| 28 |
+
"dbName",
|
| 29 |
+
"annsField",
|
| 30 |
+
"limit",
|
| 31 |
+
"filter",
|
| 32 |
+
"offset",
|
| 33 |
+
"groupingField",
|
| 34 |
+
"outputFields",
|
| 35 |
+
"searchParams",
|
| 36 |
+
"partitionNames",
|
| 37 |
+
"consistencyLevel",
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class MilvusVectorStoreConfig(BaseVectorStoreConfig):
|
| 42 |
+
"""
|
| 43 |
+
Configuration for Milvus Vector Store
|
| 44 |
+
|
| 45 |
+
This implementation uses the Azure AI Search API for vector store operations.
|
| 46 |
+
Supports vector search with embeddings generated via litellm.embeddings.
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
def __init__(self):
|
| 50 |
+
super().__init__()
|
| 51 |
+
|
| 52 |
+
def validate_environment(
|
| 53 |
+
self, headers: dict, litellm_params: Optional[GenericLiteLLMParams]
|
| 54 |
+
) -> dict:
|
| 55 |
+
api_key: Optional[str] = None
|
| 56 |
+
if litellm_params is not None:
|
| 57 |
+
api_key = litellm_params.api_key or get_secret_str("MILVUS_API_KEY")
|
| 58 |
+
|
| 59 |
+
if not api_key:
|
| 60 |
+
raise ValueError(
|
| 61 |
+
"MILVUS_API_KEY is not set. Either set it in the litellm_params or set the MILVUS_API_KEY environment variable."
|
| 62 |
+
)
|
| 63 |
+
|
| 64 |
+
headers.update({"Authorization": f"Bearer {api_key}"})
|
| 65 |
+
|
| 66 |
+
return headers
|
| 67 |
+
|
| 68 |
+
def get_auth_credentials(
|
| 69 |
+
self, litellm_params: dict
|
| 70 |
+
) -> BaseVectorStoreAuthCredentials:
|
| 71 |
+
api_key = litellm_params.get("api_key")
|
| 72 |
+
if not api_key:
|
| 73 |
+
raise ValueError(
|
| 74 |
+
"MILVUS_API_KEY is not set. Either set it in the litellm_params or set the MILVUS_API_KEY environment variable."
|
| 75 |
+
)
|
| 76 |
+
return {
|
| 77 |
+
"headers": {
|
| 78 |
+
"Authorization": f"Bearer {api_key}",
|
| 79 |
+
},
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints:
|
| 83 |
+
return {
|
| 84 |
+
"read": [
|
| 85 |
+
("POST", "/v2/vectordb/entities/search"),
|
| 86 |
+
("POST", "/v2/vectordb/entities/get"),
|
| 87 |
+
("POST", "/v2/vectordb/entities/query"),
|
| 88 |
+
],
|
| 89 |
+
"write": [
|
| 90 |
+
("POST", "/v2/vectordb/entities/upsert"),
|
| 91 |
+
("POST", "/v2/vectordb/entities/insert"),
|
| 92 |
+
],
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
def map_openai_params(
|
| 96 |
+
self, non_default_params: dict, optional_params: dict, drop_params: bool
|
| 97 |
+
) -> dict:
|
| 98 |
+
for param, value in non_default_params.items():
|
| 99 |
+
if param in MILVUS_OPTIONAL_PARAMS:
|
| 100 |
+
optional_params[param] = value
|
| 101 |
+
return optional_params
|
| 102 |
+
|
| 103 |
+
def get_complete_url(
|
| 104 |
+
self,
|
| 105 |
+
api_base: Optional[str],
|
| 106 |
+
litellm_params: dict,
|
| 107 |
+
) -> str:
|
| 108 |
+
"""
|
| 109 |
+
Get the base endpoint for Milvus API
|
| 110 |
+
|
| 111 |
+
Expected format: https://{milvus_api_base}.milvus.io
|
| 112 |
+
"""
|
| 113 |
+
api_base = api_base or get_secret_str("MILVUS_API_BASE")
|
| 114 |
+
|
| 115 |
+
if not api_base:
|
| 116 |
+
raise ValueError(
|
| 117 |
+
"Milvus API base URL is required. Set MILVUS_API_BASE environment variable or pass api_base in litellm_params."
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
if api_base:
|
| 121 |
+
return api_base.rstrip("/")
|
| 122 |
+
|
| 123 |
+
return api_base
|
| 124 |
+
|
| 125 |
+
def transform_search_vector_store_request(
|
| 126 |
+
self,
|
| 127 |
+
vector_store_id: str,
|
| 128 |
+
query: Union[str, List[str]],
|
| 129 |
+
vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
|
| 130 |
+
api_base: str,
|
| 131 |
+
litellm_logging_obj: LiteLLMLoggingObj,
|
| 132 |
+
litellm_params: dict,
|
| 133 |
+
) -> Tuple[str, Dict[str, Any]]:
|
| 134 |
+
"""
|
| 135 |
+
Transform search request for Azure AI Search API
|
| 136 |
+
|
| 137 |
+
Generates embeddings using litellm.embeddings and constructs Azure AI Search request
|
| 138 |
+
"""
|
| 139 |
+
# Convert query to string if it's a list
|
| 140 |
+
if isinstance(query, list):
|
| 141 |
+
query = " ".join(query)
|
| 142 |
+
|
| 143 |
+
# Get embedding model from litellm_params (required)
|
| 144 |
+
embedding_model = litellm_params.get("litellm_embedding_model")
|
| 145 |
+
if not embedding_model:
|
| 146 |
+
raise ValueError(
|
| 147 |
+
"embedding_model is required in litellm_params for Milvus. You can call any litellm embedding model."
|
| 148 |
+
"Example: litellm_params['embedding_model'] = 'azure/text-embedding-3-large'"
|
| 149 |
+
)
|
| 150 |
+
|
| 151 |
+
embedding_config = litellm_params.get("litellm_embedding_config", {})
|
| 152 |
+
if not embedding_config:
|
| 153 |
+
raise ValueError(
|
| 154 |
+
"embedding_config is required in litellm_params for Milvus. You can call any litellm embedding model."
|
| 155 |
+
"Example: litellm_params['embedding_config'] = {'api_base': 'https://krris-mh44uf7y-eastus2.cognitiveservices.azure.com/', 'api_key': 'os.environ/AZURE_API_KEY', 'api_version': '2025-09-01'}"
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
# Get top_k (number of results to return)
|
| 159 |
+
# Generate embedding for the query using litellm.embeddings
|
| 160 |
+
try:
|
| 161 |
+
embedding_response = litellm.embedding(
|
| 162 |
+
model=embedding_model,
|
| 163 |
+
input=[query],
|
| 164 |
+
**embedding_config,
|
| 165 |
+
)
|
| 166 |
+
query_vector = embedding_response.data[0]["embedding"]
|
| 167 |
+
except Exception as e:
|
| 168 |
+
raise Exception(f"Failed to generate embedding for query: {str(e)}")
|
| 169 |
+
|
| 170 |
+
# Azure AI Search endpoint for search
|
| 171 |
+
index_name = vector_store_id # vector_store_id is the index name
|
| 172 |
+
url = f"{api_base}/v2/vectordb/entities/search"
|
| 173 |
+
|
| 174 |
+
# Build the request body for Azure AI Search with vector search
|
| 175 |
+
request_body = {
|
| 176 |
+
"collectionName": index_name,
|
| 177 |
+
"data": [query_vector],
|
| 178 |
+
"annsField": "book_intro_vector",
|
| 179 |
+
**vector_store_search_optional_params,
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
#########################################################
|
| 183 |
+
# Update logging object with details of the request
|
| 184 |
+
#########################################################
|
| 185 |
+
litellm_logging_obj.model_call_details["input"] = query
|
| 186 |
+
litellm_logging_obj.model_call_details["embedding_model"] = embedding_model
|
| 187 |
+
|
| 188 |
+
return url, request_body
|
| 189 |
+
|
| 190 |
+
def transform_search_vector_store_response(
|
| 191 |
+
self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj
|
| 192 |
+
) -> VectorStoreSearchResponse:
|
| 193 |
+
"""
|
| 194 |
+
Transform Azure AI Search API response to standard vector store search response
|
| 195 |
+
|
| 196 |
+
Handles the format from Azure AI Search which returns:
|
| 197 |
+
{
|
| 198 |
+
"value": [
|
| 199 |
+
{
|
| 200 |
+
"id": "...",
|
| 201 |
+
"content": "...",
|
| 202 |
+
"distance": 0.95,
|
| 203 |
+
}
|
| 204 |
+
]
|
| 205 |
+
}
|
| 206 |
+
"""
|
| 207 |
+
try:
|
| 208 |
+
response_json = response.json()
|
| 209 |
+
|
| 210 |
+
# Extract results from Azure AI Search API response
|
| 211 |
+
results = response_json.get("data", [])
|
| 212 |
+
|
| 213 |
+
# Try to get text_field from optional_params first, then litellm_params
|
| 214 |
+
optional_params = litellm_logging_obj.model_call_details.get(
|
| 215 |
+
"optional_params", {}
|
| 216 |
+
)
|
| 217 |
+
text_field = optional_params.get("milvus_text_field", "")
|
| 218 |
+
|
| 219 |
+
# Fallback to litellm_params if not in optional_params
|
| 220 |
+
|
| 221 |
+
if not text_field:
|
| 222 |
+
text_field = litellm_logging_obj.model_call_details.get(
|
| 223 |
+
"litellm_params", {}
|
| 224 |
+
).get("milvus_text_field", "")
|
| 225 |
+
|
| 226 |
+
# Transform results to standard format
|
| 227 |
+
search_results: List[VectorStoreSearchResult] = []
|
| 228 |
+
for result in results:
|
| 229 |
+
# Extract text content
|
| 230 |
+
text_content = result.get(text_field, "")
|
| 231 |
+
|
| 232 |
+
content = [
|
| 233 |
+
VectorStoreResultContent(
|
| 234 |
+
text=text_content,
|
| 235 |
+
type="text",
|
| 236 |
+
)
|
| 237 |
+
]
|
| 238 |
+
|
| 239 |
+
# Get the search score (distance from the query vector)
|
| 240 |
+
score = result.get("distance", 0.0)
|
| 241 |
+
|
| 242 |
+
# Build attributes with all available metadata
|
| 243 |
+
# Exclude system fields and already-processed fields
|
| 244 |
+
attributes = {}
|
| 245 |
+
for key, value in result.items():
|
| 246 |
+
if key not in ["id", "content", "distance", text_field]:
|
| 247 |
+
attributes[key] = value
|
| 248 |
+
|
| 249 |
+
result_obj = VectorStoreSearchResult(
|
| 250 |
+
score=score,
|
| 251 |
+
content=content,
|
| 252 |
+
file_id=None,
|
| 253 |
+
filename=None,
|
| 254 |
+
attributes=attributes,
|
| 255 |
+
)
|
| 256 |
+
search_results.append(result_obj)
|
| 257 |
+
|
| 258 |
+
return VectorStoreSearchResponse(
|
| 259 |
+
object="vector_store.search_results.page",
|
| 260 |
+
search_query=litellm_logging_obj.model_call_details.get("input", ""),
|
| 261 |
+
data=search_results,
|
| 262 |
+
)
|
| 263 |
+
|
| 264 |
+
except Exception as e:
|
| 265 |
+
raise self.get_error_class(
|
| 266 |
+
error_message=str(e),
|
| 267 |
+
status_code=response.status_code,
|
| 268 |
+
headers=response.headers,
|
| 269 |
+
)
|
| 270 |
+
|
| 271 |
+
def transform_create_vector_store_request(
|
| 272 |
+
self,
|
| 273 |
+
vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams,
|
| 274 |
+
api_base: str,
|
| 275 |
+
) -> Tuple[str, Dict]:
|
| 276 |
+
raise NotImplementedError
|
| 277 |
+
|
| 278 |
+
def transform_create_vector_store_response(
|
| 279 |
+
self, response: httpx.Response
|
| 280 |
+
) -> VectorStoreCreateResponse:
|
| 281 |
+
raise NotImplementedError
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/__init__.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
MiniMax LLM Provider
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from .text_to_speech.transformation import (
|
| 6 |
+
MinimaxException,
|
| 7 |
+
MinimaxTextToSpeechConfig,
|
| 8 |
+
)
|
| 9 |
+
|
| 10 |
+
__all__ = [
|
| 11 |
+
"MinimaxTextToSpeechConfig",
|
| 12 |
+
"MinimaxException",
|
| 13 |
+
]
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/chat/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
MiniMax OpenAI-compatible chat API
|
| 3 |
+
"""
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/chat/transformation.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
MiniMax OpenAI transformation config - extends OpenAI chat config for MiniMax's OpenAI-compatible API
|
| 3 |
+
"""
|
| 4 |
+
from typing import List, Optional, Tuple
|
| 5 |
+
|
| 6 |
+
import litellm
|
| 7 |
+
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 8 |
+
from litellm.secret_managers.main import get_secret_str
|
| 9 |
+
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class MinimaxChatConfig(OpenAIGPTConfig):
|
| 13 |
+
"""
|
| 14 |
+
MiniMax OpenAI configuration that extends OpenAIGPTConfig.
|
| 15 |
+
MiniMax provides an OpenAI-compatible API at:
|
| 16 |
+
- International: https://api.minimax.io/v1
|
| 17 |
+
- China: https://api.minimaxi.com/v1
|
| 18 |
+
|
| 19 |
+
Supported models:
|
| 20 |
+
- MiniMax-M2.1
|
| 21 |
+
- MiniMax-M2.1-lightning
|
| 22 |
+
- MiniMax-M2
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
@staticmethod
|
| 26 |
+
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
|
| 27 |
+
"""
|
| 28 |
+
Get MiniMax API key from environment or parameters.
|
| 29 |
+
"""
|
| 30 |
+
return api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key
|
| 31 |
+
|
| 32 |
+
@staticmethod
|
| 33 |
+
def get_api_base(
|
| 34 |
+
api_base: Optional[str] = None,
|
| 35 |
+
) -> str:
|
| 36 |
+
"""
|
| 37 |
+
Get MiniMax API base URL.
|
| 38 |
+
Defaults to international endpoint: https://api.minimax.io/v1
|
| 39 |
+
For China, set to: https://api.minimaxi.com/v1
|
| 40 |
+
"""
|
| 41 |
+
return (
|
| 42 |
+
api_base
|
| 43 |
+
or get_secret_str("MINIMAX_API_BASE")
|
| 44 |
+
or "https://api.minimax.io/v1"
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
def get_complete_url(
|
| 48 |
+
self,
|
| 49 |
+
api_base: Optional[str],
|
| 50 |
+
api_key: Optional[str],
|
| 51 |
+
model: str,
|
| 52 |
+
optional_params: dict,
|
| 53 |
+
litellm_params: dict,
|
| 54 |
+
stream: Optional[bool] = None,
|
| 55 |
+
) -> str:
|
| 56 |
+
"""
|
| 57 |
+
Get the complete URL for MiniMax OpenAI API.
|
| 58 |
+
Override to ensure we use MiniMax's endpoint.
|
| 59 |
+
"""
|
| 60 |
+
# Get the base URL (either provided or default MiniMax endpoint)
|
| 61 |
+
base_url = self.get_api_base(api_base=api_base)
|
| 62 |
+
|
| 63 |
+
# Ensure it ends with /chat/completions
|
| 64 |
+
if base_url.endswith("/chat/completions"):
|
| 65 |
+
return base_url
|
| 66 |
+
elif base_url.endswith("/v1"):
|
| 67 |
+
return f"{base_url}/chat/completions"
|
| 68 |
+
elif base_url.endswith("/"):
|
| 69 |
+
return f"{base_url}v1/chat/completions"
|
| 70 |
+
else:
|
| 71 |
+
return f"{base_url}/v1/chat/completions"
|
| 72 |
+
|
| 73 |
+
def remove_cache_control_flag_from_messages_and_tools(
|
| 74 |
+
self,
|
| 75 |
+
model: str,
|
| 76 |
+
messages: List[AllMessageValues],
|
| 77 |
+
tools: Optional[List[ChatCompletionToolParam]] = None,
|
| 78 |
+
) -> Tuple[List[AllMessageValues], Optional[List[ChatCompletionToolParam]]]:
|
| 79 |
+
"""
|
| 80 |
+
Override to preserve cache_control for MiniMax.
|
| 81 |
+
MiniMax supports cache_control - don't strip it.
|
| 82 |
+
"""
|
| 83 |
+
# MiniMax supports cache_control, so return messages and tools unchanged
|
| 84 |
+
return messages, tools
|
| 85 |
+
|
| 86 |
+
def get_supported_openai_params(self, model: str) -> list:
|
| 87 |
+
"""
|
| 88 |
+
Get supported OpenAI parameters for MiniMax.
|
| 89 |
+
Adds reasoning_split and thinking to the list of supported params.
|
| 90 |
+
"""
|
| 91 |
+
base_params = super().get_supported_openai_params(model=model)
|
| 92 |
+
additional_params = ["reasoning_split"]
|
| 93 |
+
|
| 94 |
+
# Add thinking parameter if model supports reasoning
|
| 95 |
+
try:
|
| 96 |
+
if litellm.supports_reasoning(model=model, custom_llm_provider="minimax"):
|
| 97 |
+
additional_params.append("thinking")
|
| 98 |
+
except Exception:
|
| 99 |
+
pass
|
| 100 |
+
|
| 101 |
+
return base_params + additional_params
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/messages/transformation.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's Anthropic-compatible API
|
| 3 |
+
"""
|
| 4 |
+
from typing import Optional
|
| 5 |
+
|
| 6 |
+
import litellm
|
| 7 |
+
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
|
| 8 |
+
AnthropicMessagesConfig,
|
| 9 |
+
)
|
| 10 |
+
from litellm.secret_managers.main import get_secret_str
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class MinimaxMessagesConfig(AnthropicMessagesConfig):
|
| 14 |
+
"""
|
| 15 |
+
MiniMax Anthropic configuration that extends AnthropicConfig.
|
| 16 |
+
MiniMax provides an Anthropic-compatible API at:
|
| 17 |
+
- International: https://api.minimax.io/anthropic
|
| 18 |
+
- China: https://api.minimaxi.com/anthropic
|
| 19 |
+
|
| 20 |
+
Supported models:
|
| 21 |
+
- MiniMax-M2.1
|
| 22 |
+
- MiniMax-M2.1-lightning
|
| 23 |
+
- MiniMax-M2
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
@property
|
| 27 |
+
def custom_llm_provider(self) -> Optional[str]:
|
| 28 |
+
return "minimax"
|
| 29 |
+
|
| 30 |
+
@staticmethod
|
| 31 |
+
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
|
| 32 |
+
"""
|
| 33 |
+
Get MiniMax API key from environment or parameters.
|
| 34 |
+
"""
|
| 35 |
+
return api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key
|
| 36 |
+
|
| 37 |
+
@staticmethod
|
| 38 |
+
def get_api_base(
|
| 39 |
+
api_base: Optional[str] = None,
|
| 40 |
+
) -> str:
|
| 41 |
+
"""
|
| 42 |
+
Get MiniMax API base URL.
|
| 43 |
+
Defaults to international endpoint: https://api.minimax.io/anthropic
|
| 44 |
+
For China, set to: https://api.minimaxi.com/anthropic
|
| 45 |
+
"""
|
| 46 |
+
return (
|
| 47 |
+
api_base
|
| 48 |
+
or get_secret_str("MINIMAX_API_BASE")
|
| 49 |
+
or "https://api.minimax.io/anthropic/v1/messages"
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
def get_complete_url(
|
| 53 |
+
self,
|
| 54 |
+
api_base: Optional[str],
|
| 55 |
+
api_key: Optional[str],
|
| 56 |
+
model: str,
|
| 57 |
+
optional_params: dict,
|
| 58 |
+
litellm_params: dict,
|
| 59 |
+
stream: Optional[bool] = None,
|
| 60 |
+
) -> str:
|
| 61 |
+
"""
|
| 62 |
+
Get the complete URL for MiniMax API.
|
| 63 |
+
Override to ensure we use MiniMax's endpoint, not Anthropic's.
|
| 64 |
+
"""
|
| 65 |
+
# Get the base URL (either provided or default MiniMax endpoint)
|
| 66 |
+
base_url = self.get_api_base(api_base=api_base)
|
| 67 |
+
|
| 68 |
+
# If the base URL already includes the full path, return it
|
| 69 |
+
if base_url.endswith("/v1/messages"):
|
| 70 |
+
return base_url
|
| 71 |
+
|
| 72 |
+
# Otherwise append the messages endpoint
|
| 73 |
+
if base_url.endswith("/"):
|
| 74 |
+
return f"{base_url}v1/messages"
|
| 75 |
+
else:
|
| 76 |
+
return f"{base_url}/v1/messages"
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/text_to_speech/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
MiniMax Text-to-Speech module
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from .transformation import MinimaxException, MinimaxTextToSpeechConfig
|
| 6 |
+
|
| 7 |
+
__all__ = ["MinimaxTextToSpeechConfig", "MinimaxException"]
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/minimax/text_to_speech/transformation.py
ADDED
|
@@ -0,0 +1,414 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
MiniMax Text-to-Speech transformation
|
| 3 |
+
|
| 4 |
+
Maps OpenAI TTS spec to MiniMax TTS API (WebSocket-based HTTP API)
|
| 5 |
+
Reference: https://platform.minimax.io/docs
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import httpx
|
| 11 |
+
from httpx import Headers
|
| 12 |
+
|
| 13 |
+
import litellm
|
| 14 |
+
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
| 15 |
+
from litellm.llms.base_llm.text_to_speech.transformation import (
|
| 16 |
+
BaseTextToSpeechConfig,
|
| 17 |
+
TextToSpeechRequestData,
|
| 18 |
+
)
|
| 19 |
+
from litellm.secret_managers.main import get_secret_str
|
| 20 |
+
|
| 21 |
+
if TYPE_CHECKING:
|
| 22 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
| 23 |
+
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
| 24 |
+
else:
|
| 25 |
+
LiteLLMLoggingObj = Any
|
| 26 |
+
HttpxBinaryResponseContent = Any
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class MinimaxException(BaseLLMException):
|
| 30 |
+
"""Custom exception for MiniMax API errors"""
|
| 31 |
+
|
| 32 |
+
def __init__(
|
| 33 |
+
self,
|
| 34 |
+
status_code: int,
|
| 35 |
+
message: str,
|
| 36 |
+
headers: Optional[Union[dict, Headers]] = None,
|
| 37 |
+
):
|
| 38 |
+
super().__init__(status_code=status_code, message=message, headers=headers)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class MinimaxTextToSpeechConfig(BaseTextToSpeechConfig):
|
| 42 |
+
"""
|
| 43 |
+
Configuration for MiniMax Text-to-Speech
|
| 44 |
+
|
| 45 |
+
Reference: https://platform.minimax.io/docs
|
| 46 |
+
|
| 47 |
+
MiniMax TTS API supports both WebSocket and HTTP endpoints.
|
| 48 |
+
This implementation uses the HTTP endpoint for simplicity.
|
| 49 |
+
"""
|
| 50 |
+
|
| 51 |
+
TTS_BASE_URL = "https://api.minimax.io"
|
| 52 |
+
TTS_ENDPOINT_PATH = "/v1/t2a_v2"
|
| 53 |
+
|
| 54 |
+
# Voice mappings from OpenAI-style voices to MiniMax voice IDs
|
| 55 |
+
# MiniMax supports many voices, these are common mappings
|
| 56 |
+
VOICE_MAPPINGS = {
|
| 57 |
+
"alloy": "male-qn-qingse",
|
| 58 |
+
"echo": "male-qn-jingying",
|
| 59 |
+
"fable": "female-shaonv",
|
| 60 |
+
"onyx": "male-qn-badao",
|
| 61 |
+
"nova": "female-yujie",
|
| 62 |
+
"shimmer": "female-tianmei",
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
# Response format mappings from OpenAI to MiniMax
|
| 66 |
+
FORMAT_MAPPINGS = {
|
| 67 |
+
"mp3": "mp3",
|
| 68 |
+
"pcm": "pcm",
|
| 69 |
+
"wav": "wav",
|
| 70 |
+
"flac": "flac",
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
def get_supported_openai_params(self, model: str) -> list:
|
| 74 |
+
"""
|
| 75 |
+
MiniMax TTS supports these OpenAI parameters
|
| 76 |
+
"""
|
| 77 |
+
return ["voice", "response_format", "speed"]
|
| 78 |
+
|
| 79 |
+
def _extract_voice_id(self, voice: str) -> str:
|
| 80 |
+
"""
|
| 81 |
+
Normalize the provided voice information into a MiniMax voice_id.
|
| 82 |
+
"""
|
| 83 |
+
normalized_voice = voice.strip()
|
| 84 |
+
mapped_voice = self.VOICE_MAPPINGS.get(normalized_voice.lower())
|
| 85 |
+
return mapped_voice or normalized_voice
|
| 86 |
+
|
| 87 |
+
def _resolve_voice_id(
|
| 88 |
+
self,
|
| 89 |
+
voice: Optional[Union[str, Dict[str, Any]]],
|
| 90 |
+
params: Dict[str, Any],
|
| 91 |
+
) -> str:
|
| 92 |
+
"""
|
| 93 |
+
Determine the MiniMax voice_id based on provided voice input or parameters.
|
| 94 |
+
"""
|
| 95 |
+
mapped_voice: Optional[str] = None
|
| 96 |
+
|
| 97 |
+
if isinstance(voice, str) and voice.strip():
|
| 98 |
+
mapped_voice = self._extract_voice_id(voice)
|
| 99 |
+
elif isinstance(voice, dict):
|
| 100 |
+
for key in ("voice_id", "id", "name"):
|
| 101 |
+
candidate = voice.get(key)
|
| 102 |
+
if isinstance(candidate, str) and candidate.strip():
|
| 103 |
+
mapped_voice = self._extract_voice_id(candidate)
|
| 104 |
+
break
|
| 105 |
+
elif voice is not None:
|
| 106 |
+
mapped_voice = self._extract_voice_id(str(voice))
|
| 107 |
+
|
| 108 |
+
if mapped_voice is None:
|
| 109 |
+
voice_override = params.pop("voice_id", None)
|
| 110 |
+
if isinstance(voice_override, str) and voice_override.strip():
|
| 111 |
+
mapped_voice = self._extract_voice_id(voice_override)
|
| 112 |
+
|
| 113 |
+
if mapped_voice is None:
|
| 114 |
+
# Default to a common voice if not specified
|
| 115 |
+
mapped_voice = "male-qn-qingse"
|
| 116 |
+
|
| 117 |
+
return mapped_voice
|
| 118 |
+
|
| 119 |
+
def map_openai_params(
|
| 120 |
+
self,
|
| 121 |
+
model: str,
|
| 122 |
+
optional_params: Dict,
|
| 123 |
+
voice: Optional[Union[str, Dict]] = None,
|
| 124 |
+
drop_params: bool = False,
|
| 125 |
+
kwargs: Optional[Dict[str, Any]] = None,
|
| 126 |
+
) -> Tuple[Optional[str], Dict]:
|
| 127 |
+
"""
|
| 128 |
+
Map OpenAI parameters to MiniMax TTS parameters
|
| 129 |
+
"""
|
| 130 |
+
mapped_params: Dict[str, Any] = {}
|
| 131 |
+
|
| 132 |
+
# Work on a copy so we don't mutate the caller's dictionary
|
| 133 |
+
params = dict(optional_params) if optional_params else {}
|
| 134 |
+
|
| 135 |
+
# Extract voice identifier
|
| 136 |
+
mapped_voice = self._resolve_voice_id(voice, params)
|
| 137 |
+
|
| 138 |
+
# Response/output format
|
| 139 |
+
response_format = params.pop("response_format", None)
|
| 140 |
+
if isinstance(response_format, str):
|
| 141 |
+
mapped_format = self.FORMAT_MAPPINGS.get(response_format, "mp3")
|
| 142 |
+
mapped_params["format"] = mapped_format
|
| 143 |
+
else:
|
| 144 |
+
mapped_params["format"] = "mp3" # Default format
|
| 145 |
+
|
| 146 |
+
# Speed parameter (MiniMax supports speed from 0.5 to 2.0)
|
| 147 |
+
speed = params.pop("speed", None)
|
| 148 |
+
if speed is not None:
|
| 149 |
+
try:
|
| 150 |
+
speed_value = float(speed)
|
| 151 |
+
# Clamp speed to MiniMax's supported range
|
| 152 |
+
speed_value = max(0.5, min(2.0, speed_value))
|
| 153 |
+
mapped_params["speed"] = speed_value
|
| 154 |
+
except (TypeError, ValueError):
|
| 155 |
+
mapped_params["speed"] = 1.0
|
| 156 |
+
else:
|
| 157 |
+
mapped_params["speed"] = 1.0
|
| 158 |
+
|
| 159 |
+
# Instructions parameter is OpenAI-specific; omit to prevent API errors
|
| 160 |
+
params.pop("instructions", None)
|
| 161 |
+
|
| 162 |
+
# Store voice_id for later use in request construction
|
| 163 |
+
mapped_params["voice_id"] = mapped_voice
|
| 164 |
+
|
| 165 |
+
# Handle extra_body for additional MiniMax-specific parameters
|
| 166 |
+
extra_body = params.pop("extra_body", None)
|
| 167 |
+
if isinstance(extra_body, dict):
|
| 168 |
+
for key, value in extra_body.items():
|
| 169 |
+
if value is not None:
|
| 170 |
+
mapped_params[key] = value
|
| 171 |
+
|
| 172 |
+
# Pass through any remaining parameters
|
| 173 |
+
for key, value in params.items():
|
| 174 |
+
if value is not None:
|
| 175 |
+
mapped_params[key] = value
|
| 176 |
+
|
| 177 |
+
return mapped_voice, mapped_params
|
| 178 |
+
|
| 179 |
+
def validate_environment(
|
| 180 |
+
self,
|
| 181 |
+
headers: dict,
|
| 182 |
+
model: str,
|
| 183 |
+
api_key: Optional[str] = None,
|
| 184 |
+
api_base: Optional[str] = None,
|
| 185 |
+
) -> dict:
|
| 186 |
+
"""
|
| 187 |
+
Validate MiniMax environment and set up authentication headers
|
| 188 |
+
"""
|
| 189 |
+
api_key = api_key or litellm.api_key or get_secret_str("MINIMAX_API_KEY")
|
| 190 |
+
|
| 191 |
+
if api_key is None:
|
| 192 |
+
raise ValueError(
|
| 193 |
+
"MiniMax API key is required. Set MINIMAX_API_KEY environment variable or pass api_key parameter."
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
headers.update(
|
| 197 |
+
{
|
| 198 |
+
"Authorization": f"Bearer {api_key}",
|
| 199 |
+
"Content-Type": "application/json",
|
| 200 |
+
}
|
| 201 |
+
)
|
| 202 |
+
|
| 203 |
+
return headers
|
| 204 |
+
|
| 205 |
+
def get_error_class(
|
| 206 |
+
self, error_message: str, status_code: int, headers: Union[dict, Headers]
|
| 207 |
+
) -> BaseLLMException:
|
| 208 |
+
return MinimaxException(
|
| 209 |
+
message=error_message, status_code=status_code, headers=headers
|
| 210 |
+
)
|
| 211 |
+
|
| 212 |
+
def transform_text_to_speech_request(
|
| 213 |
+
self,
|
| 214 |
+
model: str,
|
| 215 |
+
input: str,
|
| 216 |
+
voice: Optional[str],
|
| 217 |
+
optional_params: Dict,
|
| 218 |
+
litellm_params: Dict,
|
| 219 |
+
headers: dict,
|
| 220 |
+
) -> TextToSpeechRequestData:
|
| 221 |
+
"""
|
| 222 |
+
Build the MiniMax TTS request payload.
|
| 223 |
+
|
| 224 |
+
MiniMax uses a different structure than OpenAI:
|
| 225 |
+
- model: The TTS model to use
|
| 226 |
+
- text: The input text
|
| 227 |
+
- voice_setting: Voice configuration
|
| 228 |
+
- audio_setting: Audio output configuration
|
| 229 |
+
"""
|
| 230 |
+
params = dict(optional_params) if optional_params else {}
|
| 231 |
+
|
| 232 |
+
# Extract parameters
|
| 233 |
+
voice_id = params.pop("voice_id", voice or "male-qn-qingse")
|
| 234 |
+
speed = params.pop("speed", 1.0)
|
| 235 |
+
audio_format = params.pop("format", "mp3")
|
| 236 |
+
|
| 237 |
+
# Extract additional voice settings
|
| 238 |
+
vol = params.pop("vol", 1.0) # Volume (0.1 to 10)
|
| 239 |
+
pitch = params.pop("pitch", 0) # Pitch adjustment (-12 to 12)
|
| 240 |
+
|
| 241 |
+
# Extract audio settings
|
| 242 |
+
sample_rate = params.pop("sample_rate", 32000) # 16000, 24000, 32000
|
| 243 |
+
bitrate = params.pop(
|
| 244 |
+
"bitrate", 128000
|
| 245 |
+
) # For MP3: 64000, 128000, 192000, 256000
|
| 246 |
+
channel = params.pop("channel", 1) # 1 for mono, 2 for stereo
|
| 247 |
+
|
| 248 |
+
# Output format: 'url' or 'hex' (default is 'hex')
|
| 249 |
+
output_format = params.pop("output_format", "hex")
|
| 250 |
+
|
| 251 |
+
request_body: Dict[str, Any] = {
|
| 252 |
+
"model": model,
|
| 253 |
+
"text": input,
|
| 254 |
+
"stream": False, # HTTP endpoint doesn't support streaming
|
| 255 |
+
"output_format": output_format, # 'url' or 'hex'
|
| 256 |
+
"voice_setting": {
|
| 257 |
+
"voice_id": voice_id,
|
| 258 |
+
"speed": speed,
|
| 259 |
+
"vol": vol,
|
| 260 |
+
"pitch": pitch,
|
| 261 |
+
},
|
| 262 |
+
"audio_setting": {
|
| 263 |
+
"sample_rate": sample_rate,
|
| 264 |
+
"bitrate": bitrate,
|
| 265 |
+
"format": audio_format,
|
| 266 |
+
"channel": channel,
|
| 267 |
+
},
|
| 268 |
+
}
|
| 269 |
+
|
| 270 |
+
# Handle any remaining parameters from extra_body
|
| 271 |
+
extra_body = params.pop("extra_body", None)
|
| 272 |
+
if isinstance(extra_body, dict):
|
| 273 |
+
for key, value in extra_body.items():
|
| 274 |
+
if value is not None and key not in request_body:
|
| 275 |
+
request_body[key] = value
|
| 276 |
+
|
| 277 |
+
return TextToSpeechRequestData(
|
| 278 |
+
dict_body=request_body,
|
| 279 |
+
headers={"Content-Type": "application/json"},
|
| 280 |
+
)
|
| 281 |
+
|
| 282 |
+
def transform_text_to_speech_response(
|
| 283 |
+
self,
|
| 284 |
+
model: str,
|
| 285 |
+
raw_response: httpx.Response,
|
| 286 |
+
logging_obj: LiteLLMLoggingObj,
|
| 287 |
+
) -> "HttpxBinaryResponseContent":
|
| 288 |
+
"""
|
| 289 |
+
Transform MiniMax response to standard format.
|
| 290 |
+
|
| 291 |
+
MiniMax returns JSON with base64-encoded audio data:
|
| 292 |
+
{
|
| 293 |
+
"base_resp": {"status_code": 0, "status_msg": "success"},
|
| 294 |
+
"audio_file": "<base64_encoded_audio>",
|
| 295 |
+
"extra_info": {...}
|
| 296 |
+
}
|
| 297 |
+
|
| 298 |
+
We need to decode the base64 audio and return it as binary content.
|
| 299 |
+
"""
|
| 300 |
+
import base64
|
| 301 |
+
import json
|
| 302 |
+
|
| 303 |
+
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
| 304 |
+
|
| 305 |
+
try:
|
| 306 |
+
# Parse JSON response
|
| 307 |
+
response_json = raw_response.json()
|
| 308 |
+
|
| 309 |
+
# MiniMax API response format check
|
| 310 |
+
# The API can return different structures:
|
| 311 |
+
# 1. {"data": {"audio": "..."}, "status": 0, ...} for HTTP endpoint
|
| 312 |
+
# 2. {"base_resp": {"status_code": 0, ...}, "audio_file": "..."} for older versions
|
| 313 |
+
|
| 314 |
+
# Check for errors - MiniMax uses "status" field in HTTP endpoint response
|
| 315 |
+
# status: 0 = success, 2 = invalid api key, etc.
|
| 316 |
+
status = response_json.get("status")
|
| 317 |
+
if status is not None and status != 0:
|
| 318 |
+
ced = response_json.get("ced", "Unknown error")
|
| 319 |
+
error_detail = ced if ced else f"API returned status {status}"
|
| 320 |
+
raise MinimaxException(
|
| 321 |
+
status_code=raw_response.status_code,
|
| 322 |
+
message=f"MiniMax TTS error: {error_detail}",
|
| 323 |
+
headers=dict(raw_response.headers),
|
| 324 |
+
)
|
| 325 |
+
|
| 326 |
+
# Extract audio data
|
| 327 |
+
# MiniMax returns audio in "data" field
|
| 328 |
+
data = response_json.get("data", {})
|
| 329 |
+
|
| 330 |
+
# Check if response contains a URL (output_format='url')
|
| 331 |
+
audio_url = data.get("audio_url", None)
|
| 332 |
+
if audio_url:
|
| 333 |
+
# If URL format is used, we need to fetch the audio from the URL
|
| 334 |
+
# For now, return a response indicating URL mode (TODO: fetch audio from URL)
|
| 335 |
+
raise MinimaxException(
|
| 336 |
+
status_code=500,
|
| 337 |
+
message=f"URL output format is not yet supported. Use 'hex' format or fetch from URL: {audio_url}",
|
| 338 |
+
headers=dict(raw_response.headers),
|
| 339 |
+
)
|
| 340 |
+
|
| 341 |
+
# Get hex-encoded audio data
|
| 342 |
+
audio_hex = data.get("audio", "") or response_json.get("audio_file", "")
|
| 343 |
+
|
| 344 |
+
if not audio_hex:
|
| 345 |
+
raise MinimaxException(
|
| 346 |
+
status_code=500,
|
| 347 |
+
message=f"No audio data in MiniMax response. Response keys: {list(response_json.keys())}",
|
| 348 |
+
headers=dict(raw_response.headers),
|
| 349 |
+
)
|
| 350 |
+
|
| 351 |
+
# MiniMax returns hex-encoded audio by default
|
| 352 |
+
# Try hex decoding first, fall back to base64 if that fails
|
| 353 |
+
try:
|
| 354 |
+
audio_bytes = bytes.fromhex(audio_hex)
|
| 355 |
+
except ValueError:
|
| 356 |
+
# If hex decoding fails, try base64 (for older API versions)
|
| 357 |
+
try:
|
| 358 |
+
audio_bytes = base64.b64decode(audio_hex)
|
| 359 |
+
except Exception as e:
|
| 360 |
+
raise MinimaxException(
|
| 361 |
+
status_code=500,
|
| 362 |
+
message=f"Failed to decode audio data: {str(e)}",
|
| 363 |
+
headers=dict(raw_response.headers),
|
| 364 |
+
)
|
| 365 |
+
|
| 366 |
+
# Create a new response with binary audio content
|
| 367 |
+
# We need to create a response that contains the decoded audio bytes
|
| 368 |
+
# Remove gzip encoding headers to avoid decompression issues
|
| 369 |
+
clean_headers = dict(raw_response.headers)
|
| 370 |
+
clean_headers.pop("content-encoding", None)
|
| 371 |
+
clean_headers.pop("transfer-encoding", None)
|
| 372 |
+
clean_headers["content-length"] = str(len(audio_bytes))
|
| 373 |
+
|
| 374 |
+
# Create a new response object with the binary content
|
| 375 |
+
binary_response = httpx.Response(
|
| 376 |
+
status_code=200,
|
| 377 |
+
headers=clean_headers,
|
| 378 |
+
content=audio_bytes,
|
| 379 |
+
request=raw_response.request,
|
| 380 |
+
)
|
| 381 |
+
|
| 382 |
+
return HttpxBinaryResponseContent(binary_response)
|
| 383 |
+
|
| 384 |
+
except json.JSONDecodeError as e:
|
| 385 |
+
raise MinimaxException(
|
| 386 |
+
status_code=500,
|
| 387 |
+
message=f"Failed to parse MiniMax response: {str(e)}",
|
| 388 |
+
headers=dict(raw_response.headers),
|
| 389 |
+
)
|
| 390 |
+
except Exception as e:
|
| 391 |
+
if isinstance(e, MinimaxException):
|
| 392 |
+
raise
|
| 393 |
+
raise MinimaxException(
|
| 394 |
+
status_code=500,
|
| 395 |
+
message=f"Error processing MiniMax response: {str(e)}",
|
| 396 |
+
headers=dict(raw_response.headers),
|
| 397 |
+
)
|
| 398 |
+
|
| 399 |
+
def get_complete_url(
|
| 400 |
+
self,
|
| 401 |
+
model: str,
|
| 402 |
+
api_base: Optional[str],
|
| 403 |
+
litellm_params: dict,
|
| 404 |
+
) -> str:
|
| 405 |
+
"""
|
| 406 |
+
Construct the MiniMax endpoint URL.
|
| 407 |
+
"""
|
| 408 |
+
base_url = api_base or get_secret_str("MINIMAX_API_BASE") or self.TTS_BASE_URL
|
| 409 |
+
base_url = base_url.rstrip("/")
|
| 410 |
+
|
| 411 |
+
# MiniMax uses a simple endpoint path
|
| 412 |
+
url = f"{base_url}{self.TTS_ENDPOINT_PATH}"
|
| 413 |
+
|
| 414 |
+
return url
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/audio_transcription/transformation.py
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Support for Mistral Voxtral audio transcription via ``/v1/audio/transcriptions``.
|
| 3 |
+
|
| 4 |
+
API reference: https://docs.mistral.ai/api/#tag/audio/operation/audio_transcriptions_v1_audio_transcriptions_post
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import List, Optional, Union
|
| 8 |
+
|
| 9 |
+
import httpx
|
| 10 |
+
|
| 11 |
+
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
|
| 12 |
+
from litellm.llms.base_llm.audio_transcription.transformation import (
|
| 13 |
+
AudioTranscriptionRequestData,
|
| 14 |
+
BaseAudioTranscriptionConfig,
|
| 15 |
+
)
|
| 16 |
+
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
| 17 |
+
from litellm.secret_managers.main import get_secret_str
|
| 18 |
+
from litellm.types.llms.openai import (
|
| 19 |
+
AllMessageValues,
|
| 20 |
+
OpenAIAudioTranscriptionOptionalParams,
|
| 21 |
+
)
|
| 22 |
+
from litellm.types.utils import FileTypes, TranscriptionResponse
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class MistralAudioTranscriptionException(BaseLLMException):
|
| 26 |
+
pass
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
| 30 |
+
def get_supported_openai_params(
|
| 31 |
+
self, model: str
|
| 32 |
+
) -> List[OpenAIAudioTranscriptionOptionalParams]:
|
| 33 |
+
return [
|
| 34 |
+
"language",
|
| 35 |
+
"temperature",
|
| 36 |
+
"timestamp_granularities",
|
| 37 |
+
"response_format",
|
| 38 |
+
]
|
| 39 |
+
|
| 40 |
+
def map_openai_params(
|
| 41 |
+
self,
|
| 42 |
+
non_default_params: dict,
|
| 43 |
+
optional_params: dict,
|
| 44 |
+
model: str,
|
| 45 |
+
drop_params: bool,
|
| 46 |
+
) -> dict:
|
| 47 |
+
supported_params = self.get_supported_openai_params(model)
|
| 48 |
+
for k, v in non_default_params.items():
|
| 49 |
+
if k in supported_params:
|
| 50 |
+
optional_params[k] = v
|
| 51 |
+
return optional_params
|
| 52 |
+
|
| 53 |
+
def get_complete_url(
|
| 54 |
+
self,
|
| 55 |
+
api_base: Optional[str],
|
| 56 |
+
api_key: Optional[str],
|
| 57 |
+
model: str,
|
| 58 |
+
optional_params: dict,
|
| 59 |
+
litellm_params: dict,
|
| 60 |
+
stream: Optional[bool] = None,
|
| 61 |
+
) -> str:
|
| 62 |
+
api_base = (
|
| 63 |
+
"https://api.mistral.ai/v1" if api_base is None else api_base.rstrip("/")
|
| 64 |
+
)
|
| 65 |
+
return f"{api_base}/audio/transcriptions"
|
| 66 |
+
|
| 67 |
+
def get_error_class(
|
| 68 |
+
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
| 69 |
+
) -> BaseLLMException:
|
| 70 |
+
return MistralAudioTranscriptionException(
|
| 71 |
+
message=error_message,
|
| 72 |
+
status_code=status_code,
|
| 73 |
+
headers=headers,
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
def validate_environment(
|
| 77 |
+
self,
|
| 78 |
+
headers: dict,
|
| 79 |
+
model: str,
|
| 80 |
+
messages: List[AllMessageValues],
|
| 81 |
+
optional_params: dict,
|
| 82 |
+
litellm_params: dict,
|
| 83 |
+
api_key: Optional[str] = None,
|
| 84 |
+
api_base: Optional[str] = None,
|
| 85 |
+
) -> dict:
|
| 86 |
+
if api_key is None:
|
| 87 |
+
api_key = get_secret_str("MISTRAL_API_KEY")
|
| 88 |
+
|
| 89 |
+
default_headers = {
|
| 90 |
+
"Authorization": f"Bearer {api_key}",
|
| 91 |
+
"accept": "application/json",
|
| 92 |
+
}
|
| 93 |
+
default_headers.update(headers or {})
|
| 94 |
+
return default_headers
|
| 95 |
+
|
| 96 |
+
def transform_audio_transcription_request(
|
| 97 |
+
self,
|
| 98 |
+
model: str,
|
| 99 |
+
audio_file: FileTypes,
|
| 100 |
+
optional_params: dict,
|
| 101 |
+
litellm_params: dict,
|
| 102 |
+
) -> AudioTranscriptionRequestData:
|
| 103 |
+
processed_audio = process_audio_file(audio_file)
|
| 104 |
+
|
| 105 |
+
form_fields: dict = {
|
| 106 |
+
"model": model,
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
# OpenAI-compatible params
|
| 110 |
+
for key in self.get_supported_openai_params(model):
|
| 111 |
+
value = optional_params.get(key)
|
| 112 |
+
if value is not None:
|
| 113 |
+
form_fields[key] = value
|
| 114 |
+
|
| 115 |
+
# Mistral-specific params (e.g. diarize)
|
| 116 |
+
provider_specific_params = self.get_provider_specific_params(
|
| 117 |
+
model=model,
|
| 118 |
+
optional_params=optional_params,
|
| 119 |
+
openai_params=self.get_supported_openai_params(model),
|
| 120 |
+
)
|
| 121 |
+
for key, value in provider_specific_params.items():
|
| 122 |
+
form_fields[key] = (
|
| 123 |
+
str(value).lower() if isinstance(value, bool) else str(value)
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
files = {
|
| 127 |
+
"file": (
|
| 128 |
+
processed_audio.filename,
|
| 129 |
+
processed_audio.file_content,
|
| 130 |
+
processed_audio.content_type,
|
| 131 |
+
)
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
return AudioTranscriptionRequestData(data=form_fields, files=files)
|
| 135 |
+
|
| 136 |
+
def transform_audio_transcription_response(
|
| 137 |
+
self,
|
| 138 |
+
raw_response: httpx.Response,
|
| 139 |
+
) -> TranscriptionResponse:
|
| 140 |
+
try:
|
| 141 |
+
response_json = raw_response.json()
|
| 142 |
+
except Exception:
|
| 143 |
+
raise MistralAudioTranscriptionException(
|
| 144 |
+
message=raw_response.text,
|
| 145 |
+
status_code=raw_response.status_code,
|
| 146 |
+
headers=raw_response.headers,
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
text = response_json.get("text") or ""
|
| 150 |
+
response = TranscriptionResponse(text=text)
|
| 151 |
+
|
| 152 |
+
# Preserve Mistral-specific fields (e.g. diarization segments)
|
| 153 |
+
if "segments" in response_json:
|
| 154 |
+
response["segments"] = response_json["segments"]
|
| 155 |
+
if "language" in response_json:
|
| 156 |
+
response["language"] = response_json["language"]
|
| 157 |
+
|
| 158 |
+
response._hidden_params = response_json
|
| 159 |
+
return response
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/chat/transformation.py
ADDED
|
@@ -0,0 +1,686 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transformation logic from OpenAI /v1/chat/completion format to Mistral's /chat/completion format.
|
| 3 |
+
|
| 4 |
+
Why separate file? Make it easy to see how transformation works
|
| 5 |
+
|
| 6 |
+
Docs - https://docs.mistral.ai/api/
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from typing import (
|
| 10 |
+
Any,
|
| 11 |
+
AsyncIterator,
|
| 12 |
+
Coroutine,
|
| 13 |
+
Iterator,
|
| 14 |
+
List,
|
| 15 |
+
Literal,
|
| 16 |
+
Optional,
|
| 17 |
+
Tuple,
|
| 18 |
+
Union,
|
| 19 |
+
cast,
|
| 20 |
+
get_type_hints,
|
| 21 |
+
overload,
|
| 22 |
+
)
|
| 23 |
+
|
| 24 |
+
import httpx
|
| 25 |
+
|
| 26 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
| 27 |
+
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
| 28 |
+
handle_messages_with_content_list_to_str_conversion,
|
| 29 |
+
strip_none_values_from_message,
|
| 30 |
+
)
|
| 31 |
+
from litellm.llms.openai.chat.gpt_transformation import (
|
| 32 |
+
OpenAIGPTConfig,
|
| 33 |
+
OpenAIChatCompletionStreamingHandler,
|
| 34 |
+
)
|
| 35 |
+
from litellm.secret_managers.main import get_secret_str
|
| 36 |
+
from litellm.types.llms.mistral import MistralThinkingBlock, MistralToolCallMessage
|
| 37 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 38 |
+
from litellm.types.utils import ModelResponse, ModelResponseStream
|
| 39 |
+
from litellm.utils import convert_to_model_response_object
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class MistralConfig(OpenAIGPTConfig):
|
| 43 |
+
"""
|
| 44 |
+
Reference: https://docs.mistral.ai/api/
|
| 45 |
+
|
| 46 |
+
The class `MistralConfig` provides configuration for the Mistral's Chat API interface. Below are the parameters:
|
| 47 |
+
|
| 48 |
+
- `temperature` (number or null): Defines the sampling temperature to use, varying between 0 and 2. API Default - 0.7.
|
| 49 |
+
|
| 50 |
+
- `top_p` (number or null): An alternative to sampling with temperature, used for nucleus sampling. API Default - 1.
|
| 51 |
+
|
| 52 |
+
- `max_tokens` (integer or null): This optional parameter helps to set the maximum number of tokens to generate in the chat completion. API Default - null.
|
| 53 |
+
|
| 54 |
+
- `tools` (list or null): A list of available tools for the model. Use this to specify functions for which the model can generate JSON inputs.
|
| 55 |
+
|
| 56 |
+
- `tool_choice` (string - 'auto'/'any'/'none' or null): Specifies if/how functions are called. If set to none the model won't call a function and will generate a message instead. If set to auto the model can choose to either generate a message or call a function. If set to any the model is forced to call a function. Default - 'auto'.
|
| 57 |
+
|
| 58 |
+
- `stop` (string or array of strings): Stop generation if this token is detected. Or if one of these tokens is detected when providing an array
|
| 59 |
+
|
| 60 |
+
- `random_seed` (integer or null): The seed to use for random sampling. If set, different calls will generate deterministic results.
|
| 61 |
+
|
| 62 |
+
- `safe_prompt` (boolean): Whether to inject a safety prompt before all conversations. API Default - 'false'.
|
| 63 |
+
|
| 64 |
+
- `response_format` (object or null): An object specifying the format that the model must output. Setting to { "type": "json_object" } enables JSON mode, which guarantees the message the model generates is in JSON. When using JSON mode you MUST also instruct the model to produce JSON yourself with a system or a user message.
|
| 65 |
+
"""
|
| 66 |
+
|
| 67 |
+
temperature: Optional[int] = None
|
| 68 |
+
top_p: Optional[int] = None
|
| 69 |
+
max_tokens: Optional[int] = None
|
| 70 |
+
tools: Optional[list] = None
|
| 71 |
+
tool_choice: Optional[Literal["auto", "any", "none"]] = None
|
| 72 |
+
random_seed: Optional[int] = None
|
| 73 |
+
safe_prompt: Optional[bool] = None
|
| 74 |
+
response_format: Optional[dict] = None
|
| 75 |
+
stop: Optional[Union[str, list]] = None
|
| 76 |
+
|
| 77 |
+
def __init__(
|
| 78 |
+
self,
|
| 79 |
+
temperature: Optional[int] = None,
|
| 80 |
+
top_p: Optional[int] = None,
|
| 81 |
+
max_tokens: Optional[int] = None,
|
| 82 |
+
tools: Optional[list] = None,
|
| 83 |
+
tool_choice: Optional[Literal["auto", "any", "none"]] = None,
|
| 84 |
+
random_seed: Optional[int] = None,
|
| 85 |
+
safe_prompt: Optional[bool] = None,
|
| 86 |
+
response_format: Optional[dict] = None,
|
| 87 |
+
stop: Optional[Union[str, list]] = None,
|
| 88 |
+
) -> None:
|
| 89 |
+
locals_ = locals().copy()
|
| 90 |
+
for key, value in locals_.items():
|
| 91 |
+
if key != "self" and value is not None:
|
| 92 |
+
setattr(self.__class__, key, value)
|
| 93 |
+
|
| 94 |
+
@classmethod
|
| 95 |
+
def get_config(cls):
|
| 96 |
+
return super().get_config()
|
| 97 |
+
|
| 98 |
+
def get_supported_openai_params(self, model: str) -> List[str]:
|
| 99 |
+
supported_params = [
|
| 100 |
+
"stream",
|
| 101 |
+
"temperature",
|
| 102 |
+
"top_p",
|
| 103 |
+
"max_tokens",
|
| 104 |
+
"max_completion_tokens",
|
| 105 |
+
"tools",
|
| 106 |
+
"tool_choice",
|
| 107 |
+
"seed",
|
| 108 |
+
"stop",
|
| 109 |
+
"response_format",
|
| 110 |
+
"parallel_tool_calls",
|
| 111 |
+
]
|
| 112 |
+
|
| 113 |
+
# Add reasoning support for magistral models
|
| 114 |
+
if "magistral" in model.lower():
|
| 115 |
+
supported_params.extend(["thinking", "reasoning_effort"])
|
| 116 |
+
|
| 117 |
+
return supported_params
|
| 118 |
+
|
| 119 |
+
def _map_tool_choice(self, tool_choice: str) -> str:
|
| 120 |
+
if tool_choice == "auto" or tool_choice == "none":
|
| 121 |
+
return tool_choice
|
| 122 |
+
elif tool_choice == "required":
|
| 123 |
+
return "any"
|
| 124 |
+
else: # openai 'tool_choice' object param not supported by Mistral API
|
| 125 |
+
return "any"
|
| 126 |
+
|
| 127 |
+
@staticmethod
|
| 128 |
+
def _get_mistral_reasoning_system_prompt() -> str:
|
| 129 |
+
"""
|
| 130 |
+
Returns the system prompt for Mistral reasoning models.
|
| 131 |
+
Based on Mistral's documentation: https://huggingface.co/mistralai/Magistral-Small-2506
|
| 132 |
+
|
| 133 |
+
Mistral recommends the following system prompt for reasoning:
|
| 134 |
+
"""
|
| 135 |
+
return """
|
| 136 |
+
<s>[SYSTEM_PROMPT]system_prompt
|
| 137 |
+
A user will ask you to solve a task. You should first draft your thinking process (inner monologue) until you have derived the final answer. Afterwards, write a self-contained summary of your thoughts (i.e. your summary should be succinct but contain all the critical steps you needed to reach the conclusion). You should use Markdown to format your response. Write both your thoughts and summary in the same language as the task posed by the user. NEVER use \boxed{} in your response.
|
| 138 |
+
|
| 139 |
+
Your thinking process must follow the template below:
|
| 140 |
+
<think>
|
| 141 |
+
Your thoughts or/and draft, like working through an exercise on scratch paper. Be as casual and as long as you want until you are confident to generate a correct answer.
|
| 142 |
+
</think>
|
| 143 |
+
|
| 144 |
+
Here, provide a concise summary that reflects your reasoning and presents a clear final answer to the user. Don't mention that this is a summary.
|
| 145 |
+
|
| 146 |
+
Problem:
|
| 147 |
+
|
| 148 |
+
[/SYSTEM_PROMPT][INST]user_message[/INST]<think>
|
| 149 |
+
reasoning_traces
|
| 150 |
+
</think>
|
| 151 |
+
assistant_response</s>[INST]user_message[/INST]
|
| 152 |
+
"""
|
| 153 |
+
|
| 154 |
+
def map_openai_params(
|
| 155 |
+
self,
|
| 156 |
+
non_default_params: dict,
|
| 157 |
+
optional_params: dict,
|
| 158 |
+
model: str,
|
| 159 |
+
drop_params: bool,
|
| 160 |
+
) -> dict:
|
| 161 |
+
for param, value in non_default_params.items():
|
| 162 |
+
if param == "max_tokens":
|
| 163 |
+
optional_params["max_tokens"] = value
|
| 164 |
+
if (
|
| 165 |
+
param == "max_completion_tokens"
|
| 166 |
+
): # max_completion_tokens should take priority
|
| 167 |
+
optional_params["max_tokens"] = value
|
| 168 |
+
if param == "tools":
|
| 169 |
+
# Clean tools to remove problematic schema fields for Mistral API
|
| 170 |
+
optional_params["tools"] = self._clean_tool_schema_for_mistral(value)
|
| 171 |
+
if param == "stream" and value is True:
|
| 172 |
+
optional_params["stream"] = value
|
| 173 |
+
if param == "temperature":
|
| 174 |
+
optional_params["temperature"] = value
|
| 175 |
+
if param == "top_p":
|
| 176 |
+
optional_params["top_p"] = value
|
| 177 |
+
if param == "stop":
|
| 178 |
+
optional_params["stop"] = value
|
| 179 |
+
if param == "tool_choice" and isinstance(value, str):
|
| 180 |
+
optional_params["tool_choice"] = self._map_tool_choice(
|
| 181 |
+
tool_choice=value
|
| 182 |
+
)
|
| 183 |
+
if param == "seed":
|
| 184 |
+
optional_params["extra_body"] = {"random_seed": value}
|
| 185 |
+
if param == "response_format":
|
| 186 |
+
optional_params["response_format"] = value
|
| 187 |
+
if param == "reasoning_effort" and "magistral" in model.lower():
|
| 188 |
+
# Flag that we need to add reasoning system prompt
|
| 189 |
+
optional_params["_add_reasoning_prompt"] = True
|
| 190 |
+
if param == "thinking" and "magistral" in model.lower():
|
| 191 |
+
# Flag that we need to add reasoning system prompt
|
| 192 |
+
optional_params["_add_reasoning_prompt"] = True
|
| 193 |
+
if param == "parallel_tool_calls":
|
| 194 |
+
optional_params["parallel_tool_calls"] = value
|
| 195 |
+
return optional_params
|
| 196 |
+
|
| 197 |
+
def _get_openai_compatible_provider_info(
|
| 198 |
+
self, api_base: Optional[str], api_key: Optional[str]
|
| 199 |
+
) -> Tuple[str, Optional[str]]:
|
| 200 |
+
# mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai
|
| 201 |
+
api_base = (
|
| 202 |
+
api_base
|
| 203 |
+
or get_secret_str("MISTRAL_AZURE_API_BASE") # for Azure AI Mistral
|
| 204 |
+
or "https://api.mistral.ai/v1"
|
| 205 |
+
) # type: ignore
|
| 206 |
+
|
| 207 |
+
# if api_base does not end with /v1 we add it
|
| 208 |
+
if api_base is not None and not api_base.endswith(
|
| 209 |
+
"/v1"
|
| 210 |
+
): # Mistral always needs a /v1 at the end
|
| 211 |
+
api_base = api_base + "/v1"
|
| 212 |
+
dynamic_api_key = (
|
| 213 |
+
api_key
|
| 214 |
+
or get_secret_str("MISTRAL_AZURE_API_KEY") # for Azure AI Mistral
|
| 215 |
+
or get_secret_str("MISTRAL_API_KEY")
|
| 216 |
+
)
|
| 217 |
+
return api_base, dynamic_api_key
|
| 218 |
+
|
| 219 |
+
# fmt: off
|
| 220 |
+
|
| 221 |
+
@overload
|
| 222 |
+
def _transform_messages(
|
| 223 |
+
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
|
| 224 |
+
) -> Coroutine[Any, Any, List[AllMessageValues]]:
|
| 225 |
+
...
|
| 226 |
+
|
| 227 |
+
@overload
|
| 228 |
+
def _transform_messages(
|
| 229 |
+
self,
|
| 230 |
+
messages: List[AllMessageValues],
|
| 231 |
+
model: str,
|
| 232 |
+
is_async: Literal[False] = False,
|
| 233 |
+
) -> List[AllMessageValues]:
|
| 234 |
+
...
|
| 235 |
+
# fmt: on
|
| 236 |
+
|
| 237 |
+
def _transform_messages(
|
| 238 |
+
self, messages: List[AllMessageValues], model: str, is_async: bool = False
|
| 239 |
+
) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
|
| 240 |
+
"""
|
| 241 |
+
- handles scenario where content is list and not string
|
| 242 |
+
- content list is just text, and no images
|
| 243 |
+
- if image passed in, then just return as is (user-intended)
|
| 244 |
+
- if `name` is passed, then drop it for mistral API: https://github.com/BerriAI/litellm/issues/6696
|
| 245 |
+
|
| 246 |
+
Motivation: mistral api doesn't support content as a list.
|
| 247 |
+
The above statement is not valid now. Need to plan to remove all the #1,2,3
|
| 248 |
+
Mistral API supports content as a list.
|
| 249 |
+
"""
|
| 250 |
+
## 1. If 'image_url' or 'file' in content, then transform with base class and mistral-specific handling
|
| 251 |
+
for m in messages:
|
| 252 |
+
_content_block = m.get("content")
|
| 253 |
+
if _content_block and isinstance(_content_block, list):
|
| 254 |
+
if any(c.get("type") in ["image_url", "file"] for c in _content_block):
|
| 255 |
+
if is_async:
|
| 256 |
+
return self._transform_messages_async(messages, model)
|
| 257 |
+
else:
|
| 258 |
+
messages = self._transform_messages_sync(messages, model)
|
| 259 |
+
return messages
|
| 260 |
+
|
| 261 |
+
## 2. If content is list, then convert to string
|
| 262 |
+
messages = handle_messages_with_content_list_to_str_conversion(messages)
|
| 263 |
+
|
| 264 |
+
## 3. Handle name in message
|
| 265 |
+
new_messages: List[AllMessageValues] = []
|
| 266 |
+
for m in messages:
|
| 267 |
+
m = MistralConfig._handle_name_in_message(m)
|
| 268 |
+
m = MistralConfig._handle_tool_call_message(m)
|
| 269 |
+
if MistralConfig._is_empty_assistant_message(m):
|
| 270 |
+
continue
|
| 271 |
+
m = strip_none_values_from_message(m) # prevents 'extra_forbidden' error
|
| 272 |
+
new_messages.append(m)
|
| 273 |
+
|
| 274 |
+
if is_async:
|
| 275 |
+
return super()._transform_messages(new_messages, model, True)
|
| 276 |
+
else:
|
| 277 |
+
return super()._transform_messages(new_messages, model, False)
|
| 278 |
+
|
| 279 |
+
async def _transform_messages_async(
|
| 280 |
+
self, messages: List[AllMessageValues], model: str
|
| 281 |
+
) -> List[AllMessageValues]:
|
| 282 |
+
"""
|
| 283 |
+
Handle modification of messages for Mistral API in an async context.
|
| 284 |
+
"""
|
| 285 |
+
# Call parent async method to handle basic transformations
|
| 286 |
+
# and then apply Mistral-specific handling for files
|
| 287 |
+
messages = await super()._transform_messages(messages, model, True)
|
| 288 |
+
messages = self._handle_message_with_file(messages)
|
| 289 |
+
return messages
|
| 290 |
+
|
| 291 |
+
def _transform_messages_sync(
|
| 292 |
+
self, messages: List[AllMessageValues], model: str
|
| 293 |
+
) -> List[AllMessageValues]:
|
| 294 |
+
"""Handle modification of messages for Mistral API in a sync context."""
|
| 295 |
+
# Call parent sync method to handle basic transformations
|
| 296 |
+
# and then apply Mistral-specific handling for files
|
| 297 |
+
# This is the sync version of the async method above
|
| 298 |
+
messages = super()._transform_messages(messages, model, False)
|
| 299 |
+
messages = self._handle_message_with_file(messages)
|
| 300 |
+
return messages
|
| 301 |
+
|
| 302 |
+
def _handle_message_with_file(
|
| 303 |
+
self, messages: List[AllMessageValues]
|
| 304 |
+
) -> List[AllMessageValues]:
|
| 305 |
+
"""
|
| 306 |
+
Mistral API supports only 'file_id' in message content with type 'file'.
|
| 307 |
+
"""
|
| 308 |
+
for m in messages:
|
| 309 |
+
_content_block = m.get("content")
|
| 310 |
+
if _content_block and isinstance(_content_block, list):
|
| 311 |
+
if any(c.get("type") == "file" for c in _content_block):
|
| 312 |
+
# If file content is present, we get file_id from 'file' attribute of content block
|
| 313 |
+
# then replace 'file' with 'file_id' and assign the value of 'file_id' attribute to it.
|
| 314 |
+
file_contents = [
|
| 315 |
+
c for c in _content_block if c.get("type") == "file"
|
| 316 |
+
]
|
| 317 |
+
for file_content in file_contents:
|
| 318 |
+
file_id = file_content.get("file", {}).get("file_id")
|
| 319 |
+
if file_id:
|
| 320 |
+
# Replace 'file' with 'file_id'
|
| 321 |
+
file_content["file_id"] = file_id # type: ignore
|
| 322 |
+
file_content.pop("file", None)
|
| 323 |
+
return messages
|
| 324 |
+
|
| 325 |
+
def _add_reasoning_system_prompt_if_needed(
|
| 326 |
+
self, messages: List[AllMessageValues], optional_params: dict
|
| 327 |
+
) -> List[AllMessageValues]:
|
| 328 |
+
"""
|
| 329 |
+
Add reasoning system prompt for Mistral magistral models when reasoning_effort is specified.
|
| 330 |
+
"""
|
| 331 |
+
if not optional_params.get("_add_reasoning_prompt", False):
|
| 332 |
+
return messages
|
| 333 |
+
|
| 334 |
+
# Check if there's already a system message
|
| 335 |
+
has_system_message = any(msg.get("role") == "system" for msg in messages)
|
| 336 |
+
|
| 337 |
+
if has_system_message:
|
| 338 |
+
# Prepend reasoning instructions to existing system message
|
| 339 |
+
for i, msg in enumerate(messages):
|
| 340 |
+
if msg.get("role") == "system":
|
| 341 |
+
existing_content = msg.get("content", "")
|
| 342 |
+
reasoning_prompt = self._get_mistral_reasoning_system_prompt()
|
| 343 |
+
|
| 344 |
+
# Handle both string and list content, preserving original format
|
| 345 |
+
if isinstance(existing_content, str):
|
| 346 |
+
# String content - prepend reasoning prompt
|
| 347 |
+
new_content: Union[
|
| 348 |
+
str, list
|
| 349 |
+
] = f"{reasoning_prompt}\n\n{existing_content}"
|
| 350 |
+
elif isinstance(existing_content, list):
|
| 351 |
+
# List content - prepend reasoning prompt as text block
|
| 352 |
+
new_content = [
|
| 353 |
+
{"type": "text", "text": reasoning_prompt + "\n\n"}
|
| 354 |
+
] + existing_content
|
| 355 |
+
else:
|
| 356 |
+
# Fallback for any other type - convert to string
|
| 357 |
+
new_content = f"{reasoning_prompt}\n\n{str(existing_content)}"
|
| 358 |
+
|
| 359 |
+
messages[i] = cast(
|
| 360 |
+
AllMessageValues, {**msg, "content": new_content}
|
| 361 |
+
)
|
| 362 |
+
break
|
| 363 |
+
else:
|
| 364 |
+
# Add new system message with reasoning instructions
|
| 365 |
+
reasoning_message: AllMessageValues = cast(
|
| 366 |
+
AllMessageValues,
|
| 367 |
+
{
|
| 368 |
+
"role": "system",
|
| 369 |
+
"content": self._get_mistral_reasoning_system_prompt(),
|
| 370 |
+
},
|
| 371 |
+
)
|
| 372 |
+
messages = [reasoning_message] + messages
|
| 373 |
+
|
| 374 |
+
# Remove the internal flag
|
| 375 |
+
optional_params.pop("_add_reasoning_prompt", None)
|
| 376 |
+
return messages
|
| 377 |
+
|
| 378 |
+
@classmethod
|
| 379 |
+
def _clean_tool_schema_for_mistral(cls, tools: list) -> list:
|
| 380 |
+
"""
|
| 381 |
+
Clean tool schemas to remove fields that cause issues with Mistral API.
|
| 382 |
+
|
| 383 |
+
Removes:
|
| 384 |
+
- $id and $schema fields (cause grammar validation errors)
|
| 385 |
+
- additionalProperties=False (causes OpenAI API schema errors)
|
| 386 |
+
- strict field (not supported by Mistral)
|
| 387 |
+
|
| 388 |
+
Args:
|
| 389 |
+
tools: List of tool definitions
|
| 390 |
+
max_depth: Maximum recursion depth for schema cleaning (default: 10)
|
| 391 |
+
|
| 392 |
+
Returns:
|
| 393 |
+
Cleaned tools list
|
| 394 |
+
"""
|
| 395 |
+
if not tools:
|
| 396 |
+
return tools
|
| 397 |
+
|
| 398 |
+
import copy
|
| 399 |
+
|
| 400 |
+
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
| 401 |
+
from litellm.utils import _remove_json_schema_refs
|
| 402 |
+
|
| 403 |
+
cleaned_tools = copy.deepcopy(tools)
|
| 404 |
+
|
| 405 |
+
# Apply all cleaning functions with max_depth protection
|
| 406 |
+
cleaned_tools = _remove_json_schema_refs(
|
| 407 |
+
cleaned_tools, max_depth=DEFAULT_MAX_RECURSE_DEPTH
|
| 408 |
+
)
|
| 409 |
+
|
| 410 |
+
return cleaned_tools
|
| 411 |
+
|
| 412 |
+
@classmethod
|
| 413 |
+
def _handle_name_in_message(cls, message: AllMessageValues) -> AllMessageValues:
|
| 414 |
+
"""
|
| 415 |
+
Mistral API only supports `name` in tool messages
|
| 416 |
+
|
| 417 |
+
If role == tool, then we keep `name` if it's not an empty string
|
| 418 |
+
Otherwise, we drop `name`
|
| 419 |
+
"""
|
| 420 |
+
_name = message.get("name") # type: ignore
|
| 421 |
+
|
| 422 |
+
if _name is not None:
|
| 423 |
+
# Remove name if not a tool message
|
| 424 |
+
if message["role"] != "tool":
|
| 425 |
+
message.pop("name", None) # type: ignore
|
| 426 |
+
# For tool messages, remove name if it's an empty string
|
| 427 |
+
elif isinstance(_name, str) and len(_name.strip()) == 0:
|
| 428 |
+
message.pop("name", None) # type: ignore
|
| 429 |
+
|
| 430 |
+
return message
|
| 431 |
+
|
| 432 |
+
@classmethod
|
| 433 |
+
def _handle_tool_call_message(cls, message: AllMessageValues) -> AllMessageValues:
|
| 434 |
+
"""
|
| 435 |
+
Mistral API only supports tool_calls in Messages in `MistralToolCallMessage` spec
|
| 436 |
+
"""
|
| 437 |
+
_tool_calls = message.get("tool_calls")
|
| 438 |
+
mistral_tool_calls: List[MistralToolCallMessage] = []
|
| 439 |
+
if _tool_calls is not None and isinstance(_tool_calls, list):
|
| 440 |
+
for _tool in _tool_calls:
|
| 441 |
+
_tool_call_message = MistralToolCallMessage(
|
| 442 |
+
id=_tool.get("id"),
|
| 443 |
+
type="function",
|
| 444 |
+
function=_tool.get("function"), # type: ignore
|
| 445 |
+
)
|
| 446 |
+
mistral_tool_calls.append(_tool_call_message)
|
| 447 |
+
message["tool_calls"] = mistral_tool_calls # type: ignore
|
| 448 |
+
return message
|
| 449 |
+
|
| 450 |
+
@classmethod
|
| 451 |
+
def _is_empty_assistant_message(cls, message: AllMessageValues) -> bool:
|
| 452 |
+
"""
|
| 453 |
+
Mistral API does not support empty string in assistant content.
|
| 454 |
+
"""
|
| 455 |
+
from litellm.types.llms.openai import ChatCompletionAssistantMessage
|
| 456 |
+
|
| 457 |
+
set_keys = get_type_hints(ChatCompletionAssistantMessage).keys()
|
| 458 |
+
|
| 459 |
+
all_expected_values_are_empty = True
|
| 460 |
+
for key in set_keys:
|
| 461 |
+
if key != "role" and message.get(key) is not None:
|
| 462 |
+
if key == "content" and message.get(key) == "":
|
| 463 |
+
continue
|
| 464 |
+
else:
|
| 465 |
+
all_expected_values_are_empty = False
|
| 466 |
+
break
|
| 467 |
+
return all_expected_values_are_empty
|
| 468 |
+
|
| 469 |
+
@staticmethod
|
| 470 |
+
def _handle_empty_content_response(response_data: dict) -> dict:
|
| 471 |
+
"""
|
| 472 |
+
Handle Mistral-specific behavior where empty string content should be converted to None.
|
| 473 |
+
|
| 474 |
+
Mistral API sometimes returns empty string content ('') instead of null,
|
| 475 |
+
which can cause issues with downstream processing.
|
| 476 |
+
|
| 477 |
+
Args:
|
| 478 |
+
response_data: The raw response data from Mistral API
|
| 479 |
+
|
| 480 |
+
Returns:
|
| 481 |
+
dict: The response data with empty string content converted to None
|
| 482 |
+
"""
|
| 483 |
+
if response_data.get("choices") and len(response_data["choices"]) > 0:
|
| 484 |
+
for choice in response_data["choices"]:
|
| 485 |
+
if choice.get("message") and choice["message"].get("content") == "":
|
| 486 |
+
choice["message"]["content"] = None
|
| 487 |
+
return response_data
|
| 488 |
+
|
| 489 |
+
@staticmethod
|
| 490 |
+
def _convert_thinking_block_to_reasoning_content(
|
| 491 |
+
thinking_blocks: MistralThinkingBlock,
|
| 492 |
+
) -> str:
|
| 493 |
+
"""
|
| 494 |
+
Convert Mistral thinking blocks to reasoning content.
|
| 495 |
+
"""
|
| 496 |
+
return "\n".join(
|
| 497 |
+
[block.get("text", "") for block in thinking_blocks["thinking"]]
|
| 498 |
+
)
|
| 499 |
+
|
| 500 |
+
@staticmethod
|
| 501 |
+
def _handle_content_list_to_str_conversion(response_data: dict) -> dict:
|
| 502 |
+
"""
|
| 503 |
+
Handle Mistral's content list format and extract thinking content.
|
| 504 |
+
|
| 505 |
+
Map mistral's content list to string and extract thinking blocks:
|
| 506 |
+
- Thinking block -> reasoning_content field
|
| 507 |
+
- Text block -> content field
|
| 508 |
+
"""
|
| 509 |
+
|
| 510 |
+
if response_data.get("choices") and len(response_data["choices"]) > 0:
|
| 511 |
+
for choice in response_data["choices"]:
|
| 512 |
+
if choice.get("message") and choice["message"].get("content"):
|
| 513 |
+
content = choice["message"]["content"]
|
| 514 |
+
|
| 515 |
+
# Only process if content is a list
|
| 516 |
+
if isinstance(content, list):
|
| 517 |
+
thinking_content = ""
|
| 518 |
+
text_content = ""
|
| 519 |
+
|
| 520 |
+
# Process each content block
|
| 521 |
+
for block in content:
|
| 522 |
+
if block.get("type") == "thinking":
|
| 523 |
+
thinking_blocks = block.get("thinking", [])
|
| 524 |
+
thinking_texts = []
|
| 525 |
+
for thinking_block in thinking_blocks:
|
| 526 |
+
if thinking_block.get("type") == "text":
|
| 527 |
+
thinking_texts.append(
|
| 528 |
+
thinking_block.get("text", "")
|
| 529 |
+
)
|
| 530 |
+
thinking_content = "\n".join(thinking_texts)
|
| 531 |
+
elif block.get("type") == "text":
|
| 532 |
+
text_content = block.get("text", "")
|
| 533 |
+
|
| 534 |
+
# Set the extracted content
|
| 535 |
+
choice["message"]["content"] = text_content
|
| 536 |
+
if thinking_content:
|
| 537 |
+
choice["message"]["reasoning_content"] = thinking_content
|
| 538 |
+
|
| 539 |
+
return response_data
|
| 540 |
+
|
| 541 |
+
def transform_request(
|
| 542 |
+
self,
|
| 543 |
+
model: str,
|
| 544 |
+
messages: List[AllMessageValues],
|
| 545 |
+
optional_params: dict,
|
| 546 |
+
litellm_params: dict,
|
| 547 |
+
headers: dict,
|
| 548 |
+
) -> dict:
|
| 549 |
+
"""
|
| 550 |
+
Transform the overall request to be sent to the API.
|
| 551 |
+
For magistral models, adds reasoning system prompt when reasoning_effort is specified.
|
| 552 |
+
|
| 553 |
+
Returns:
|
| 554 |
+
dict: The transformed request. Sent as the body of the API call.
|
| 555 |
+
"""
|
| 556 |
+
# Add reasoning system prompt if needed (for magistral models)
|
| 557 |
+
if "magistral" in model.lower() and optional_params.get(
|
| 558 |
+
"_add_reasoning_prompt", False
|
| 559 |
+
):
|
| 560 |
+
messages = self._add_reasoning_system_prompt_if_needed(
|
| 561 |
+
messages, optional_params
|
| 562 |
+
)
|
| 563 |
+
|
| 564 |
+
# Call parent transform_request which handles _transform_messages
|
| 565 |
+
return super().transform_request(
|
| 566 |
+
model=model,
|
| 567 |
+
messages=messages,
|
| 568 |
+
optional_params=optional_params,
|
| 569 |
+
litellm_params=litellm_params,
|
| 570 |
+
headers=headers,
|
| 571 |
+
)
|
| 572 |
+
|
| 573 |
+
def transform_response(
|
| 574 |
+
self,
|
| 575 |
+
model: str,
|
| 576 |
+
raw_response: httpx.Response,
|
| 577 |
+
model_response: ModelResponse,
|
| 578 |
+
logging_obj: LiteLLMLoggingObj,
|
| 579 |
+
request_data: dict,
|
| 580 |
+
messages: List[AllMessageValues],
|
| 581 |
+
optional_params: dict,
|
| 582 |
+
litellm_params: dict,
|
| 583 |
+
encoding: Any,
|
| 584 |
+
api_key: Optional[str] = None,
|
| 585 |
+
json_mode: Optional[bool] = None,
|
| 586 |
+
) -> ModelResponse:
|
| 587 |
+
"""
|
| 588 |
+
Transform the raw response from Mistral API.
|
| 589 |
+
Handles Mistral-specific behavior like converting empty string content to None
|
| 590 |
+
and extracting thinking content from content lists.
|
| 591 |
+
"""
|
| 592 |
+
logging_obj.post_call(original_response=raw_response.text)
|
| 593 |
+
logging_obj.model_call_details["response_headers"] = raw_response.headers
|
| 594 |
+
|
| 595 |
+
# Handle Mistral-specific response transformations
|
| 596 |
+
response_data = raw_response.json()
|
| 597 |
+
response_data = self._handle_empty_content_response(response_data)
|
| 598 |
+
response_data = self._handle_content_list_to_str_conversion(response_data)
|
| 599 |
+
|
| 600 |
+
final_response_obj = cast(
|
| 601 |
+
ModelResponse,
|
| 602 |
+
convert_to_model_response_object(
|
| 603 |
+
response_object=response_data,
|
| 604 |
+
model_response_object=model_response,
|
| 605 |
+
hidden_params={"headers": raw_response.headers},
|
| 606 |
+
_response_headers=dict(raw_response.headers),
|
| 607 |
+
),
|
| 608 |
+
)
|
| 609 |
+
|
| 610 |
+
return final_response_obj
|
| 611 |
+
|
| 612 |
+
def get_model_response_iterator(
|
| 613 |
+
self,
|
| 614 |
+
streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
|
| 615 |
+
sync_stream: bool,
|
| 616 |
+
json_mode: Optional[bool] = False,
|
| 617 |
+
):
|
| 618 |
+
return MistralChatResponseIterator(
|
| 619 |
+
streaming_response=streaming_response,
|
| 620 |
+
sync_stream=sync_stream,
|
| 621 |
+
json_mode=json_mode,
|
| 622 |
+
)
|
| 623 |
+
|
| 624 |
+
|
| 625 |
+
class MistralChatResponseIterator(OpenAIChatCompletionStreamingHandler):
|
| 626 |
+
def chunk_parser(self, chunk: dict) -> ModelResponseStream:
|
| 627 |
+
try:
|
| 628 |
+
for choice in chunk.get("choices", []):
|
| 629 |
+
delta = choice.get("delta", {})
|
| 630 |
+
content = delta.get("content")
|
| 631 |
+
if isinstance(content, list):
|
| 632 |
+
(
|
| 633 |
+
normalized_text,
|
| 634 |
+
thinking_blocks,
|
| 635 |
+
reasoning_content,
|
| 636 |
+
) = self._normalize_content_blocks(content)
|
| 637 |
+
delta["content"] = normalized_text
|
| 638 |
+
if thinking_blocks:
|
| 639 |
+
delta["thinking_blocks"] = thinking_blocks
|
| 640 |
+
delta["reasoning_content"] = reasoning_content
|
| 641 |
+
else:
|
| 642 |
+
delta.pop("thinking_blocks", None)
|
| 643 |
+
delta.pop("reasoning_content", None)
|
| 644 |
+
except Exception:
|
| 645 |
+
# Fall back to default parsing if custom handling fails
|
| 646 |
+
return super().chunk_parser(chunk)
|
| 647 |
+
|
| 648 |
+
return super().chunk_parser(chunk)
|
| 649 |
+
|
| 650 |
+
@staticmethod
|
| 651 |
+
def _normalize_content_blocks(
|
| 652 |
+
content_blocks: List[dict],
|
| 653 |
+
) -> Tuple[Optional[str], List[dict], Optional[str]]:
|
| 654 |
+
"""
|
| 655 |
+
Convert Mistral magistral content blocks into OpenAI-compatible content + thinking_blocks.
|
| 656 |
+
"""
|
| 657 |
+
text_segments: List[str] = []
|
| 658 |
+
thinking_blocks: List[dict] = []
|
| 659 |
+
reasoning_segments: List[str] = []
|
| 660 |
+
|
| 661 |
+
for block in content_blocks:
|
| 662 |
+
block_type = block.get("type")
|
| 663 |
+
if block_type == "thinking":
|
| 664 |
+
mistral_thinking = block.get("thinking", [])
|
| 665 |
+
thinking_text_parts: List[str] = []
|
| 666 |
+
for thinking_block in mistral_thinking:
|
| 667 |
+
if thinking_block.get("type") == "text":
|
| 668 |
+
thinking_text_parts.append(thinking_block.get("text", ""))
|
| 669 |
+
thinking_text = "".join(thinking_text_parts)
|
| 670 |
+
if thinking_text:
|
| 671 |
+
reasoning_segments.append(thinking_text)
|
| 672 |
+
thinking_blocks.append(
|
| 673 |
+
{
|
| 674 |
+
"type": "thinking",
|
| 675 |
+
"thinking": thinking_text,
|
| 676 |
+
"signature": "mistral",
|
| 677 |
+
}
|
| 678 |
+
)
|
| 679 |
+
elif block_type == "text":
|
| 680 |
+
text_segments.append(block.get("text", ""))
|
| 681 |
+
|
| 682 |
+
normalized_text = "".join(text_segments) if text_segments else None
|
| 683 |
+
reasoning_content = (
|
| 684 |
+
"\n".join(reasoning_segments) if reasoning_segments else None
|
| 685 |
+
)
|
| 686 |
+
return normalized_text, thinking_blocks, reasoning_content
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/embedding.py
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Calls handled in openai/
|
| 3 |
+
as mistral is an openai-compatible endpoint.
|
| 4 |
+
"""
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/mistral_embedding_transformation.py
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Mistral OCR transformation module."""
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/guardrail_translation/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Mistral OCR handler for Unified Guardrails."""
|
| 2 |
+
|
| 3 |
+
from litellm.llms.mistral.ocr.guardrail_translation.handler import OCRHandler
|
| 4 |
+
from litellm.types.utils import CallTypes
|
| 5 |
+
|
| 6 |
+
guardrail_translation_mappings = {
|
| 7 |
+
CallTypes.ocr: OCRHandler,
|
| 8 |
+
CallTypes.aocr: OCRHandler,
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
__all__ = ["guardrail_translation_mappings", "OCRHandler"]
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/guardrail_translation/handler.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
OCR Handler for Unified Guardrails
|
| 3 |
+
|
| 4 |
+
Provides guardrail translation support for the OCR endpoint.
|
| 5 |
+
Processes the extracted markdown text from OCR pages.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import TYPE_CHECKING, Any, List, Optional
|
| 9 |
+
|
| 10 |
+
from litellm._logging import verbose_proxy_logger
|
| 11 |
+
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
|
| 12 |
+
from litellm.types.utils import GenericGuardrailAPIInputs
|
| 13 |
+
|
| 14 |
+
if TYPE_CHECKING:
|
| 15 |
+
from litellm.integrations.custom_guardrail import CustomGuardrail
|
| 16 |
+
from litellm.llms.base_llm.ocr.transformation import OCRResponse
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class OCRHandler(BaseTranslation):
|
| 20 |
+
"""
|
| 21 |
+
Handler for processing OCR requests/responses with guardrails.
|
| 22 |
+
|
| 23 |
+
Input: The OCR input is a document URL/reference - not text content.
|
| 24 |
+
We pass the document URL as text for guardrails that may want to
|
| 25 |
+
validate or filter document sources.
|
| 26 |
+
|
| 27 |
+
Output: OCR responses contain extracted markdown text per page.
|
| 28 |
+
The handler extracts all page markdown, applies guardrails,
|
| 29 |
+
and maps the guardrailed text back to the pages.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
async def process_input_messages(
|
| 33 |
+
self,
|
| 34 |
+
data: dict,
|
| 35 |
+
guardrail_to_apply: "CustomGuardrail",
|
| 36 |
+
litellm_logging_obj: Optional[Any] = None,
|
| 37 |
+
) -> Any:
|
| 38 |
+
"""
|
| 39 |
+
Process OCR input by applying guardrails to the document reference.
|
| 40 |
+
|
| 41 |
+
The OCR input contains a document dict with a URL. We extract
|
| 42 |
+
the URL and pass it to the guardrail for validation.
|
| 43 |
+
|
| 44 |
+
Args:
|
| 45 |
+
data: Request data containing 'document' parameter
|
| 46 |
+
guardrail_to_apply: The guardrail instance to apply
|
| 47 |
+
litellm_logging_obj: Optional logging object
|
| 48 |
+
|
| 49 |
+
Returns:
|
| 50 |
+
Modified data with guardrails applied
|
| 51 |
+
"""
|
| 52 |
+
document = data.get("document")
|
| 53 |
+
if document is None or not isinstance(document, dict):
|
| 54 |
+
verbose_proxy_logger.debug(
|
| 55 |
+
"OCR guardrail: No valid document found in request data"
|
| 56 |
+
)
|
| 57 |
+
return data
|
| 58 |
+
|
| 59 |
+
# Extract the document URL for guardrail checking
|
| 60 |
+
texts_to_check: List[str] = []
|
| 61 |
+
doc_type = document.get("type")
|
| 62 |
+
if doc_type == "document_url":
|
| 63 |
+
url = document.get("document_url")
|
| 64 |
+
if url and isinstance(url, str):
|
| 65 |
+
texts_to_check.append(url)
|
| 66 |
+
elif doc_type == "image_url":
|
| 67 |
+
url = document.get("image_url")
|
| 68 |
+
if url and isinstance(url, str):
|
| 69 |
+
texts_to_check.append(url)
|
| 70 |
+
|
| 71 |
+
if not texts_to_check:
|
| 72 |
+
return data
|
| 73 |
+
|
| 74 |
+
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
| 75 |
+
model = data.get("model")
|
| 76 |
+
if model:
|
| 77 |
+
inputs["model"] = model
|
| 78 |
+
|
| 79 |
+
await guardrail_to_apply.apply_guardrail(
|
| 80 |
+
inputs=inputs,
|
| 81 |
+
request_data=data,
|
| 82 |
+
input_type="request",
|
| 83 |
+
logging_obj=litellm_logging_obj,
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
return data
|
| 87 |
+
|
| 88 |
+
async def process_output_response(
|
| 89 |
+
self,
|
| 90 |
+
response: "OCRResponse",
|
| 91 |
+
guardrail_to_apply: "CustomGuardrail",
|
| 92 |
+
litellm_logging_obj: Optional[Any] = None,
|
| 93 |
+
user_api_key_dict: Optional[Any] = None,
|
| 94 |
+
) -> Any:
|
| 95 |
+
"""
|
| 96 |
+
Process OCR output by applying guardrails to extracted page text.
|
| 97 |
+
|
| 98 |
+
Extracts markdown text from each OCR page, applies guardrails,
|
| 99 |
+
and maps the guardrailed text back to the pages.
|
| 100 |
+
|
| 101 |
+
Args:
|
| 102 |
+
response: OCRResponse with pages containing markdown text
|
| 103 |
+
guardrail_to_apply: The guardrail instance to apply
|
| 104 |
+
litellm_logging_obj: Optional logging object
|
| 105 |
+
user_api_key_dict: User API key metadata
|
| 106 |
+
|
| 107 |
+
Returns:
|
| 108 |
+
Modified OCRResponse with guardrailed page text
|
| 109 |
+
"""
|
| 110 |
+
if not hasattr(response, "pages") or not response.pages:
|
| 111 |
+
verbose_proxy_logger.debug("OCR guardrail: No pages found in OCR response")
|
| 112 |
+
return response
|
| 113 |
+
|
| 114 |
+
# Extract markdown text from all pages
|
| 115 |
+
texts_to_check: List[str] = []
|
| 116 |
+
page_indices: List[int] = []
|
| 117 |
+
for i, page in enumerate(response.pages):
|
| 118 |
+
if hasattr(page, "markdown") and page.markdown:
|
| 119 |
+
texts_to_check.append(page.markdown)
|
| 120 |
+
page_indices.append(i)
|
| 121 |
+
|
| 122 |
+
if not texts_to_check:
|
| 123 |
+
return response
|
| 124 |
+
|
| 125 |
+
inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
|
| 126 |
+
model = getattr(response, "model", None)
|
| 127 |
+
if model:
|
| 128 |
+
inputs["model"] = model
|
| 129 |
+
|
| 130 |
+
# Add user metadata if available
|
| 131 |
+
if user_api_key_dict is not None:
|
| 132 |
+
metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
|
| 133 |
+
inputs.update(metadata) # type: ignore
|
| 134 |
+
|
| 135 |
+
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
|
| 136 |
+
inputs=inputs,
|
| 137 |
+
request_data={},
|
| 138 |
+
input_type="response",
|
| 139 |
+
logging_obj=litellm_logging_obj,
|
| 140 |
+
)
|
| 141 |
+
|
| 142 |
+
# Map guardrailed text back to pages
|
| 143 |
+
guardrailed_texts = guardrailed_inputs.get("texts", [])
|
| 144 |
+
for idx, page_idx in enumerate(page_indices):
|
| 145 |
+
if idx < len(guardrailed_texts):
|
| 146 |
+
response.pages[page_idx].markdown = guardrailed_texts[idx]
|
| 147 |
+
|
| 148 |
+
verbose_proxy_logger.debug(
|
| 149 |
+
"OCR guardrail: Applied guardrail to %d pages",
|
| 150 |
+
len(guardrailed_texts),
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
return response
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/mistral/ocr/transformation.py
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Mistral OCR transformation implementation.
|
| 3 |
+
"""
|
| 4 |
+
from typing import Any, Dict, Optional
|
| 5 |
+
|
| 6 |
+
import httpx
|
| 7 |
+
|
| 8 |
+
from litellm._logging import verbose_logger
|
| 9 |
+
from litellm.llms.base_llm.ocr.transformation import (
|
| 10 |
+
BaseOCRConfig,
|
| 11 |
+
DocumentType,
|
| 12 |
+
OCRRequestData,
|
| 13 |
+
OCRResponse,
|
| 14 |
+
)
|
| 15 |
+
from litellm.secret_managers.main import get_secret_str
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class MistralOCRConfig(BaseOCRConfig):
|
| 19 |
+
"""
|
| 20 |
+
Mistral OCR transformation configuration.
|
| 21 |
+
|
| 22 |
+
Reference: https://docs.mistral.ai/api/#tag/ocr
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
def __init__(self) -> None:
|
| 26 |
+
super().__init__()
|
| 27 |
+
|
| 28 |
+
def get_supported_ocr_params(self, model: str) -> list:
|
| 29 |
+
"""
|
| 30 |
+
Get supported OCR parameters for Mistral OCR.
|
| 31 |
+
|
| 32 |
+
Mistral OCR supports:
|
| 33 |
+
- pages: List of page numbers to process
|
| 34 |
+
- include_image_base64: Whether to include base64 encoded images
|
| 35 |
+
- image_limit: Maximum number of images to return
|
| 36 |
+
- image_min_size: Minimum size of images to include
|
| 37 |
+
- bbox_annotation_format: Format for bounding box annotations
|
| 38 |
+
- document_annotation_format: Format for document annotations
|
| 39 |
+
- extract_header: Whether to extract document header
|
| 40 |
+
- extract_footer: Whether to extract document footer
|
| 41 |
+
"""
|
| 42 |
+
return [
|
| 43 |
+
"pages",
|
| 44 |
+
"include_image_base64",
|
| 45 |
+
"image_limit",
|
| 46 |
+
"image_min_size",
|
| 47 |
+
"bbox_annotation_format",
|
| 48 |
+
"document_annotation_format",
|
| 49 |
+
"extract_header",
|
| 50 |
+
"extract_footer",
|
| 51 |
+
]
|
| 52 |
+
|
| 53 |
+
def map_ocr_params(
|
| 54 |
+
self,
|
| 55 |
+
non_default_params: dict,
|
| 56 |
+
optional_params: dict,
|
| 57 |
+
model: str,
|
| 58 |
+
) -> dict:
|
| 59 |
+
"""
|
| 60 |
+
Map OCR parameters to Mistral-specific format.
|
| 61 |
+
|
| 62 |
+
Mistral accepts these parameters directly, so no transformation needed.
|
| 63 |
+
Just filter out unsupported params.
|
| 64 |
+
"""
|
| 65 |
+
supported_params = self.get_supported_ocr_params(model=model)
|
| 66 |
+
|
| 67 |
+
# Only include params that are in the supported list
|
| 68 |
+
mapped_params = {}
|
| 69 |
+
for param, value in non_default_params.items():
|
| 70 |
+
if param in supported_params:
|
| 71 |
+
mapped_params[param] = value
|
| 72 |
+
|
| 73 |
+
return mapped_params
|
| 74 |
+
|
| 75 |
+
def validate_environment(
|
| 76 |
+
self,
|
| 77 |
+
headers: Dict,
|
| 78 |
+
model: str,
|
| 79 |
+
api_key: Optional[str] = None,
|
| 80 |
+
api_base: Optional[str] = None,
|
| 81 |
+
litellm_params: Optional[dict] = None,
|
| 82 |
+
**kwargs,
|
| 83 |
+
) -> Dict:
|
| 84 |
+
"""
|
| 85 |
+
Validate environment and return headers for Mistral OCR.
|
| 86 |
+
"""
|
| 87 |
+
# Get API key from environment if not provided
|
| 88 |
+
if api_key is None:
|
| 89 |
+
api_key = get_secret_str("MISTRAL_API_KEY")
|
| 90 |
+
|
| 91 |
+
if api_key is None:
|
| 92 |
+
raise ValueError(
|
| 93 |
+
"Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params"
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
headers = {
|
| 97 |
+
"Authorization": f"Bearer {api_key}",
|
| 98 |
+
**headers,
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
# Don't set Content-Type for multipart/form-data - httpx will handle it
|
| 102 |
+
|
| 103 |
+
return headers
|
| 104 |
+
|
| 105 |
+
def get_complete_url(
|
| 106 |
+
self,
|
| 107 |
+
api_base: Optional[str],
|
| 108 |
+
model: str,
|
| 109 |
+
optional_params: dict,
|
| 110 |
+
litellm_params: Optional[dict] = None,
|
| 111 |
+
**kwargs,
|
| 112 |
+
) -> str:
|
| 113 |
+
"""
|
| 114 |
+
Get complete URL for Mistral OCR endpoint.
|
| 115 |
+
|
| 116 |
+
Returns: https://api.mistral.ai/v1/ocr
|
| 117 |
+
"""
|
| 118 |
+
if api_base is None:
|
| 119 |
+
api_base = "https://api.mistral.ai/v1"
|
| 120 |
+
|
| 121 |
+
# Ensure no trailing slash
|
| 122 |
+
api_base = api_base.rstrip("/")
|
| 123 |
+
|
| 124 |
+
# Remove /v1 if it's already in the base to avoid duplication
|
| 125 |
+
if api_base.endswith("/v1"):
|
| 126 |
+
return f"{api_base}/ocr"
|
| 127 |
+
|
| 128 |
+
return f"{api_base}/v1/ocr"
|
| 129 |
+
|
| 130 |
+
def transform_ocr_request(
|
| 131 |
+
self,
|
| 132 |
+
model: str,
|
| 133 |
+
document: DocumentType,
|
| 134 |
+
optional_params: dict,
|
| 135 |
+
headers: dict,
|
| 136 |
+
**kwargs,
|
| 137 |
+
) -> OCRRequestData:
|
| 138 |
+
"""
|
| 139 |
+
Transform OCR request to Mistral-specific format.
|
| 140 |
+
|
| 141 |
+
Mistral OCR API accepts:
|
| 142 |
+
{
|
| 143 |
+
"model": "mistral-ocr-latest",
|
| 144 |
+
"document": {
|
| 145 |
+
"type": "document_url",
|
| 146 |
+
"document_url": "<https-url or data-uri>"
|
| 147 |
+
},
|
| 148 |
+
"pages": [0], # optional
|
| 149 |
+
"include_image_base64": false, # optional
|
| 150 |
+
...
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
Args:
|
| 154 |
+
model: Model name (e.g., "mistral-ocr-latest")
|
| 155 |
+
document: Document dict from user (Mistral format) - already validated in main.py
|
| 156 |
+
optional_params: Already mapped optional parameters
|
| 157 |
+
headers: Request headers
|
| 158 |
+
|
| 159 |
+
Returns:
|
| 160 |
+
OCRRequestData with JSON data
|
| 161 |
+
"""
|
| 162 |
+
verbose_logger.debug(f"Mistral OCR transform_ocr_request - model: {model}")
|
| 163 |
+
|
| 164 |
+
# Document parameter is the Mistral-format dict from the user
|
| 165 |
+
# Just pass it through as-is to the Mistral API
|
| 166 |
+
if not isinstance(document, dict):
|
| 167 |
+
raise ValueError(f"Expected document dict, got {type(document)}")
|
| 168 |
+
|
| 169 |
+
# Build request data - use document dict directly
|
| 170 |
+
data = {
|
| 171 |
+
"model": model,
|
| 172 |
+
"document": document, # Pass through the Mistral-format document dict
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
+
# Add all optional parameters from the already-mapped optional_params
|
| 176 |
+
data.update(optional_params)
|
| 177 |
+
|
| 178 |
+
# No multipart files - using JSON
|
| 179 |
+
return OCRRequestData(data=data, files=None)
|
| 180 |
+
|
| 181 |
+
def transform_ocr_response(
|
| 182 |
+
self,
|
| 183 |
+
model: str,
|
| 184 |
+
raw_response: httpx.Response,
|
| 185 |
+
logging_obj: Any,
|
| 186 |
+
**kwargs,
|
| 187 |
+
) -> OCRResponse:
|
| 188 |
+
"""
|
| 189 |
+
Return Mistral OCR response in native format.
|
| 190 |
+
|
| 191 |
+
Mistral OCR is the standard format for LiteLLM OCR responses.
|
| 192 |
+
No transformation needed - return native response.
|
| 193 |
+
|
| 194 |
+
Mistral OCR returns:
|
| 195 |
+
{
|
| 196 |
+
"pages": [
|
| 197 |
+
{
|
| 198 |
+
"index": 0,
|
| 199 |
+
"markdown": "extracted text content",
|
| 200 |
+
"images": [...],
|
| 201 |
+
"dimensions": {...}
|
| 202 |
+
},
|
| 203 |
+
...
|
| 204 |
+
],
|
| 205 |
+
"model": "mistral-ocr-2505-completion",
|
| 206 |
+
"document_annotation": null,
|
| 207 |
+
"usage_info": {...}
|
| 208 |
+
}
|
| 209 |
+
"""
|
| 210 |
+
try:
|
| 211 |
+
response_json = raw_response.json()
|
| 212 |
+
|
| 213 |
+
verbose_logger.debug(f"Mistral OCR response keys: {response_json.keys()}")
|
| 214 |
+
|
| 215 |
+
# Return native Mistral format - no transformation
|
| 216 |
+
return OCRResponse(
|
| 217 |
+
pages=response_json.get("pages", []),
|
| 218 |
+
model=response_json.get("model", model),
|
| 219 |
+
document_annotation=response_json.get("document_annotation"),
|
| 220 |
+
usage_info=response_json.get("usage_info"),
|
| 221 |
+
object="ocr",
|
| 222 |
+
)
|
| 223 |
+
except Exception as e:
|
| 224 |
+
verbose_logger.error(f"Error parsing Mistral OCR response: {e}")
|
| 225 |
+
raise e
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/moonshot/chat/transformation.py
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Translates from OpenAI's `/v1/chat/completions` to Moonshot AI's `/v1/chat/completions`
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload
|
| 6 |
+
|
| 7 |
+
import litellm
|
| 8 |
+
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
| 9 |
+
handle_messages_with_content_list_to_str_conversion,
|
| 10 |
+
)
|
| 11 |
+
from litellm.secret_managers.main import get_secret_str
|
| 12 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 13 |
+
from litellm.utils import supports_reasoning
|
| 14 |
+
|
| 15 |
+
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class MoonshotChatConfig(OpenAIGPTConfig):
|
| 19 |
+
@overload
|
| 20 |
+
def _transform_messages(
|
| 21 |
+
self, messages: List[AllMessageValues], model: str, is_async: Literal[True]
|
| 22 |
+
) -> Coroutine[Any, Any, List[AllMessageValues]]:
|
| 23 |
+
...
|
| 24 |
+
|
| 25 |
+
@overload
|
| 26 |
+
def _transform_messages(
|
| 27 |
+
self,
|
| 28 |
+
messages: List[AllMessageValues],
|
| 29 |
+
model: str,
|
| 30 |
+
is_async: Literal[False] = False,
|
| 31 |
+
) -> List[AllMessageValues]:
|
| 32 |
+
...
|
| 33 |
+
|
| 34 |
+
def _transform_messages(
|
| 35 |
+
self, messages: List[AllMessageValues], model: str, is_async: bool = False
|
| 36 |
+
) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
|
| 37 |
+
"""
|
| 38 |
+
Moonshot text-only models don't support content in list format.
|
| 39 |
+
Multimodal models (kimi-k2.5, kimi-latest, etc.) accept the
|
| 40 |
+
standard OpenAI content array with non-text blocks (image_url,
|
| 41 |
+
input_audio, video_url, file, etc.).
|
| 42 |
+
|
| 43 |
+
If any message contains a non-text content part, skip flattening
|
| 44 |
+
so the multimodal payload is preserved.
|
| 45 |
+
"""
|
| 46 |
+
has_non_text = False
|
| 47 |
+
for m in messages:
|
| 48 |
+
_content = m.get("content")
|
| 49 |
+
if _content and isinstance(_content, list):
|
| 50 |
+
if any(c.get("type") != "text" for c in _content):
|
| 51 |
+
has_non_text = True
|
| 52 |
+
break
|
| 53 |
+
|
| 54 |
+
if not has_non_text:
|
| 55 |
+
messages = handle_messages_with_content_list_to_str_conversion(messages)
|
| 56 |
+
|
| 57 |
+
if is_async:
|
| 58 |
+
return super()._transform_messages(
|
| 59 |
+
messages=messages, model=model, is_async=True
|
| 60 |
+
)
|
| 61 |
+
else:
|
| 62 |
+
return super()._transform_messages(
|
| 63 |
+
messages=messages, model=model, is_async=False
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
def _get_openai_compatible_provider_info(
|
| 67 |
+
self, api_base: Optional[str], api_key: Optional[str]
|
| 68 |
+
) -> Tuple[Optional[str], Optional[str]]:
|
| 69 |
+
api_base = api_base or get_secret_str("MOONSHOT_API_BASE") or "https://api.moonshot.ai/v1" # type: ignore
|
| 70 |
+
dynamic_api_key = api_key or get_secret_str("MOONSHOT_API_KEY")
|
| 71 |
+
return api_base, dynamic_api_key
|
| 72 |
+
|
| 73 |
+
def get_complete_url(
|
| 74 |
+
self,
|
| 75 |
+
api_base: Optional[str],
|
| 76 |
+
api_key: Optional[str],
|
| 77 |
+
model: str,
|
| 78 |
+
optional_params: dict,
|
| 79 |
+
litellm_params: dict,
|
| 80 |
+
stream: Optional[bool] = None,
|
| 81 |
+
) -> str:
|
| 82 |
+
"""
|
| 83 |
+
If api_base is not provided, use the default Moonshot AI /chat/completions endpoint.
|
| 84 |
+
"""
|
| 85 |
+
if not api_base:
|
| 86 |
+
api_base = "https://api.moonshot.ai/v1"
|
| 87 |
+
|
| 88 |
+
if not api_base.endswith("/chat/completions"):
|
| 89 |
+
api_base = f"{api_base}/chat/completions"
|
| 90 |
+
|
| 91 |
+
return api_base
|
| 92 |
+
|
| 93 |
+
def get_supported_openai_params(self, model: str) -> list:
|
| 94 |
+
"""
|
| 95 |
+
Get the supported OpenAI params for Moonshot AI models
|
| 96 |
+
|
| 97 |
+
Moonshot AI limitations:
|
| 98 |
+
- functions parameter is not supported (use tools instead)
|
| 99 |
+
- tool_choice doesn't support "required" value
|
| 100 |
+
- kimi-thinking-preview doesn't support tool calls at all
|
| 101 |
+
"""
|
| 102 |
+
excluded_params: List[str] = ["functions"]
|
| 103 |
+
|
| 104 |
+
# kimi-thinking-preview has additional limitations
|
| 105 |
+
if "kimi-thinking-preview" in model:
|
| 106 |
+
excluded_params.extend(["tools", "tool_choice"])
|
| 107 |
+
|
| 108 |
+
base_openai_params = super().get_supported_openai_params(model=model)
|
| 109 |
+
final_params: List[str] = []
|
| 110 |
+
for param in base_openai_params:
|
| 111 |
+
if param not in excluded_params:
|
| 112 |
+
final_params.append(param)
|
| 113 |
+
|
| 114 |
+
return final_params
|
| 115 |
+
|
| 116 |
+
def map_openai_params(
|
| 117 |
+
self,
|
| 118 |
+
non_default_params: dict,
|
| 119 |
+
optional_params: dict,
|
| 120 |
+
model: str,
|
| 121 |
+
drop_params: bool,
|
| 122 |
+
) -> dict:
|
| 123 |
+
"""
|
| 124 |
+
Map OpenAI parameters to Moonshot AI parameters
|
| 125 |
+
|
| 126 |
+
Handles Moonshot AI specific limitations:
|
| 127 |
+
- tool_choice doesn't support "required" value
|
| 128 |
+
- Temperature <0.3 limitation for n>1
|
| 129 |
+
"""
|
| 130 |
+
supported_openai_params = self.get_supported_openai_params(model)
|
| 131 |
+
for param, value in non_default_params.items():
|
| 132 |
+
if param == "max_completion_tokens":
|
| 133 |
+
optional_params["max_tokens"] = value
|
| 134 |
+
elif param in supported_openai_params:
|
| 135 |
+
optional_params[param] = value
|
| 136 |
+
|
| 137 |
+
##########################################
|
| 138 |
+
# temperature limitations
|
| 139 |
+
# 1. `temperature` on KIMI API is [0, 1] but OpenAI is [0, 2]
|
| 140 |
+
# 2. If temperature < 0.3 and n > 1, KIMI will raise an exception.
|
| 141 |
+
# If we enter this condition, we set the temperature to 0.3 as suggested by Moonshot AI
|
| 142 |
+
##########################################
|
| 143 |
+
if "temperature" in optional_params:
|
| 144 |
+
if optional_params["temperature"] > 1:
|
| 145 |
+
optional_params["temperature"] = 1
|
| 146 |
+
if optional_params["temperature"] < 0.3 and optional_params.get("n", 1) > 1:
|
| 147 |
+
optional_params["temperature"] = 0.3
|
| 148 |
+
return optional_params
|
| 149 |
+
|
| 150 |
+
def fill_reasoning_content(
|
| 151 |
+
self, messages: List[AllMessageValues]
|
| 152 |
+
) -> List[AllMessageValues]:
|
| 153 |
+
"""
|
| 154 |
+
Moonshot reasoning models require `reasoning_content` on every assistant
|
| 155 |
+
message that contains tool_calls (multi-turn tool-calling flows).
|
| 156 |
+
|
| 157 |
+
For each such message that is missing the field:
|
| 158 |
+
1. Check if reasoning_content exists at the top level (for Pydantic models
|
| 159 |
+
that have the attribute but don't support 'in' operator)
|
| 160 |
+
2. Promote provider_specific_fields["reasoning_content"] if present and non-empty
|
| 161 |
+
(this is where LiteLLM stores it from a previous response)
|
| 162 |
+
3. Otherwise inject a single space — the minimum value the API accepts
|
| 163 |
+
Messages that already carry the field, or are not assistant/tool-call messages,
|
| 164 |
+
are appended as-is (no copy made).
|
| 165 |
+
"""
|
| 166 |
+
result: List[AllMessageValues] = []
|
| 167 |
+
for msg in messages:
|
| 168 |
+
if (
|
| 169 |
+
msg.get("role") == "assistant"
|
| 170 |
+
and msg.get("tool_calls")
|
| 171 |
+
and not msg.get(
|
| 172 |
+
"reasoning_content"
|
| 173 |
+
) # Check using .get() which works for both dicts and Pydantic models
|
| 174 |
+
):
|
| 175 |
+
patched = dict(cast(dict, msg))
|
| 176 |
+
provider_fields = patched.get("provider_specific_fields") or {}
|
| 177 |
+
stored = provider_fields.get("reasoning_content")
|
| 178 |
+
if stored:
|
| 179 |
+
patched["reasoning_content"] = stored
|
| 180 |
+
# Remove the promoted key from provider_specific_fields to
|
| 181 |
+
# avoid sending the value twice in the serialised request body
|
| 182 |
+
cleaned_provider_fields = dict(provider_fields)
|
| 183 |
+
cleaned_provider_fields.pop("reasoning_content", None)
|
| 184 |
+
patched["provider_specific_fields"] = cleaned_provider_fields
|
| 185 |
+
else:
|
| 186 |
+
litellm.verbose_logger.warning(
|
| 187 |
+
"Moonshot reasoning model: assistant tool-call message is missing "
|
| 188 |
+
"`reasoning_content`. Injecting a placeholder to satisfy API validation. "
|
| 189 |
+
"For best results, preserve `reasoning_content` from the original "
|
| 190 |
+
"assistant response when building multi-turn conversation history."
|
| 191 |
+
)
|
| 192 |
+
patched["reasoning_content"] = " "
|
| 193 |
+
result.append(cast(AllMessageValues, patched))
|
| 194 |
+
else:
|
| 195 |
+
result.append(msg)
|
| 196 |
+
return result
|
| 197 |
+
|
| 198 |
+
def transform_request(
|
| 199 |
+
self,
|
| 200 |
+
model: str,
|
| 201 |
+
messages: List[AllMessageValues],
|
| 202 |
+
optional_params: dict,
|
| 203 |
+
litellm_params: dict,
|
| 204 |
+
headers: dict,
|
| 205 |
+
) -> dict:
|
| 206 |
+
"""
|
| 207 |
+
Transform the overall request to be sent to the API.
|
| 208 |
+
Returns:
|
| 209 |
+
dict: The transformed request. Sent as the body of the API call.
|
| 210 |
+
"""
|
| 211 |
+
# Add tool_choice="required" message if needed
|
| 212 |
+
if optional_params.get("tool_choice", None) == "required":
|
| 213 |
+
messages = self._add_tool_choice_required_message(
|
| 214 |
+
messages=messages,
|
| 215 |
+
optional_params=optional_params,
|
| 216 |
+
)
|
| 217 |
+
|
| 218 |
+
# Moonshot reasoning models: fill in reasoning_content before the API call
|
| 219 |
+
if supports_reasoning(model=model, custom_llm_provider="moonshot"):
|
| 220 |
+
messages = self.fill_reasoning_content(messages)
|
| 221 |
+
|
| 222 |
+
# Call parent transform_request which handles _transform_messages
|
| 223 |
+
return super().transform_request(
|
| 224 |
+
model=model,
|
| 225 |
+
messages=messages,
|
| 226 |
+
optional_params=optional_params,
|
| 227 |
+
litellm_params=litellm_params,
|
| 228 |
+
headers=headers,
|
| 229 |
+
)
|
| 230 |
+
|
| 231 |
+
def _add_tool_choice_required_message(
|
| 232 |
+
self, messages: List[AllMessageValues], optional_params: dict
|
| 233 |
+
) -> List[AllMessageValues]:
|
| 234 |
+
"""
|
| 235 |
+
Add a message to the messages list to indicate that the tool choice is required.
|
| 236 |
+
|
| 237 |
+
https://platform.moonshot.ai/docs/guide/migrating-from-openai-to-kimi#about-tool_choice
|
| 238 |
+
"""
|
| 239 |
+
messages.append(
|
| 240 |
+
{
|
| 241 |
+
"role": "user",
|
| 242 |
+
"content": "Please select a tool to handle the current issue.", # Usually, the Kimi large language model understands the intention to invoke a tool and selects one for invocation
|
| 243 |
+
}
|
| 244 |
+
)
|
| 245 |
+
optional_params.pop("tool_choice")
|
| 246 |
+
return messages
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/__init__.py
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/chat/__init__.py
ADDED
|
File without changes
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/morph/chat/transformation.py
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Transform request from OpenAI format to Morph format.
|
| 3 |
+
|
| 4 |
+
[TODO] Docs: Morph supports the OpenAI API format.
|
| 5 |
+
https://docs.morphllm.com/quickstart
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import Optional, Tuple
|
| 9 |
+
|
| 10 |
+
from litellm.secret_managers.main import get_secret_str
|
| 11 |
+
|
| 12 |
+
from ...openai_like.chat.transformation import OpenAILikeChatConfig
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class MorphChatConfig(OpenAILikeChatConfig):
|
| 16 |
+
"""
|
| 17 |
+
Transform request from OpenAI format to Morph format.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
@property
|
| 21 |
+
def custom_llm_provider(self) -> Optional[str]:
|
| 22 |
+
return "morph"
|
| 23 |
+
|
| 24 |
+
def _get_openai_compatible_provider_info(
|
| 25 |
+
self, api_base: Optional[str], api_key: Optional[str]
|
| 26 |
+
) -> Tuple[Optional[str], Optional[str]]:
|
| 27 |
+
api_base = (
|
| 28 |
+
api_base
|
| 29 |
+
or get_secret_str("MORPH_API_BASE")
|
| 30 |
+
or "https://api.morphllm.com/v1" # default api base
|
| 31 |
+
)
|
| 32 |
+
dynamic_api_key = api_key or get_secret_str("MORPH_API_KEY")
|
| 33 |
+
return api_base, dynamic_api_key
|
| 34 |
+
|
| 35 |
+
def get_supported_openai_params(self, model: str) -> list:
|
| 36 |
+
return [
|
| 37 |
+
"messages",
|
| 38 |
+
"model",
|
| 39 |
+
"stream",
|
| 40 |
+
]
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nebius/chat/transformation.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Nebius AI Studio Chat Completions API - Transformation
|
| 3 |
+
|
| 4 |
+
This is OpenAI compatible - no translation needed / occurs
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class NebiusConfig(OpenAIGPTConfig):
|
| 11 |
+
def map_openai_params(
|
| 12 |
+
self,
|
| 13 |
+
non_default_params: dict,
|
| 14 |
+
optional_params: dict,
|
| 15 |
+
model: str,
|
| 16 |
+
drop_params: bool,
|
| 17 |
+
) -> dict:
|
| 18 |
+
"""
|
| 19 |
+
map max_completion_tokens param to max_tokens
|
| 20 |
+
"""
|
| 21 |
+
supported_openai_params = self.get_supported_openai_params(model=model)
|
| 22 |
+
for param, value in non_default_params.items():
|
| 23 |
+
if param == "max_completion_tokens":
|
| 24 |
+
optional_params["max_tokens"] = value
|
| 25 |
+
elif param in supported_openai_params:
|
| 26 |
+
optional_params[param] = value
|
| 27 |
+
return optional_params
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nebius/embedding/transformation.py
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Calls handled in openai/
|
| 3 |
+
|
| 4 |
+
as Nebius AI Studio is an openai-compatible endpoint.
|
| 5 |
+
"""
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/chat/handler.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from typing import Callable, Optional, Union
|
| 3 |
+
|
| 4 |
+
import litellm
|
| 5 |
+
from litellm.llms.custom_httpx.http_handler import (
|
| 6 |
+
AsyncHTTPHandler,
|
| 7 |
+
HTTPHandler,
|
| 8 |
+
_get_httpx_client,
|
| 9 |
+
)
|
| 10 |
+
from litellm.utils import ModelResponse
|
| 11 |
+
|
| 12 |
+
from .transformation import NLPCloudConfig
|
| 13 |
+
|
| 14 |
+
nlp_config = NLPCloudConfig()
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def completion(
|
| 18 |
+
model: str,
|
| 19 |
+
messages: list,
|
| 20 |
+
api_base: str,
|
| 21 |
+
model_response: ModelResponse,
|
| 22 |
+
print_verbose: Callable,
|
| 23 |
+
encoding,
|
| 24 |
+
api_key,
|
| 25 |
+
logging_obj,
|
| 26 |
+
optional_params: dict,
|
| 27 |
+
litellm_params: dict,
|
| 28 |
+
logger_fn=None,
|
| 29 |
+
default_max_tokens_to_sample=None,
|
| 30 |
+
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
|
| 31 |
+
headers={},
|
| 32 |
+
):
|
| 33 |
+
headers = nlp_config.validate_environment(
|
| 34 |
+
api_key=api_key,
|
| 35 |
+
headers=headers,
|
| 36 |
+
model=model,
|
| 37 |
+
messages=messages,
|
| 38 |
+
optional_params=optional_params,
|
| 39 |
+
litellm_params=litellm_params,
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
## Load Config
|
| 43 |
+
config = litellm.NLPCloudConfig.get_config()
|
| 44 |
+
for k, v in config.items():
|
| 45 |
+
if (
|
| 46 |
+
k not in optional_params
|
| 47 |
+
): # completion(top_k=3) > togetherai_config(top_k=3) <- allows for dynamic variables to be passed in
|
| 48 |
+
optional_params[k] = v
|
| 49 |
+
|
| 50 |
+
completion_url_fragment_1 = api_base
|
| 51 |
+
completion_url_fragment_2 = "/generation"
|
| 52 |
+
model = model
|
| 53 |
+
|
| 54 |
+
completion_url = completion_url_fragment_1 + model + completion_url_fragment_2
|
| 55 |
+
data = nlp_config.transform_request(
|
| 56 |
+
model=model,
|
| 57 |
+
messages=messages,
|
| 58 |
+
optional_params=optional_params,
|
| 59 |
+
litellm_params=litellm_params,
|
| 60 |
+
headers=headers,
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
## LOGGING
|
| 64 |
+
logging_obj.pre_call(
|
| 65 |
+
input=None,
|
| 66 |
+
api_key=api_key,
|
| 67 |
+
additional_args={
|
| 68 |
+
"complete_input_dict": data,
|
| 69 |
+
"headers": headers,
|
| 70 |
+
"api_base": completion_url,
|
| 71 |
+
},
|
| 72 |
+
)
|
| 73 |
+
## COMPLETION CALL
|
| 74 |
+
if client is None or not isinstance(client, HTTPHandler):
|
| 75 |
+
client = _get_httpx_client()
|
| 76 |
+
|
| 77 |
+
response = client.post(
|
| 78 |
+
completion_url,
|
| 79 |
+
headers=headers,
|
| 80 |
+
data=json.dumps(data),
|
| 81 |
+
stream=optional_params["stream"] if "stream" in optional_params else False,
|
| 82 |
+
)
|
| 83 |
+
if "stream" in optional_params and optional_params["stream"] is True:
|
| 84 |
+
return clean_and_iterate_chunks(response)
|
| 85 |
+
else:
|
| 86 |
+
return nlp_config.transform_response(
|
| 87 |
+
model=model,
|
| 88 |
+
raw_response=response,
|
| 89 |
+
model_response=model_response,
|
| 90 |
+
logging_obj=logging_obj,
|
| 91 |
+
api_key=api_key,
|
| 92 |
+
request_data=data,
|
| 93 |
+
messages=messages,
|
| 94 |
+
optional_params=optional_params,
|
| 95 |
+
litellm_params=litellm_params,
|
| 96 |
+
encoding=encoding,
|
| 97 |
+
)
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
# def clean_and_iterate_chunks(response):
|
| 101 |
+
# def process_chunk(chunk):
|
| 102 |
+
# print(f"received chunk: {chunk}")
|
| 103 |
+
# cleaned_chunk = chunk.decode("utf-8")
|
| 104 |
+
# # Perform further processing based on your needs
|
| 105 |
+
# return cleaned_chunk
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
# for line in response.iter_lines():
|
| 109 |
+
# if line:
|
| 110 |
+
# yield process_chunk(line)
|
| 111 |
+
def clean_and_iterate_chunks(response):
|
| 112 |
+
buffer = b""
|
| 113 |
+
|
| 114 |
+
for chunk in response.iter_content(chunk_size=1024):
|
| 115 |
+
if not chunk:
|
| 116 |
+
break
|
| 117 |
+
|
| 118 |
+
buffer += chunk
|
| 119 |
+
while b"\x00" in buffer:
|
| 120 |
+
buffer = buffer.replace(b"\x00", b"")
|
| 121 |
+
yield buffer.decode("utf-8")
|
| 122 |
+
buffer = b""
|
| 123 |
+
|
| 124 |
+
# No more data expected, yield any remaining data in the buffer
|
| 125 |
+
if buffer:
|
| 126 |
+
yield buffer.decode("utf-8")
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def embedding():
|
| 130 |
+
# logic for parsing in - calling - parsing out model embedding calls
|
| 131 |
+
pass
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/chat/transformation.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
import time
|
| 3 |
+
from typing import TYPE_CHECKING, Any, List, Optional, Union
|
| 4 |
+
|
| 5 |
+
import httpx
|
| 6 |
+
|
| 7 |
+
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
| 8 |
+
convert_content_list_to_str,
|
| 9 |
+
)
|
| 10 |
+
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
| 11 |
+
from litellm.types.llms.openai import AllMessageValues
|
| 12 |
+
from litellm.utils import ModelResponse, Usage
|
| 13 |
+
|
| 14 |
+
from ..common_utils import NLPCloudError
|
| 15 |
+
|
| 16 |
+
if TYPE_CHECKING:
|
| 17 |
+
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
| 18 |
+
|
| 19 |
+
LoggingClass = LiteLLMLoggingObj
|
| 20 |
+
else:
|
| 21 |
+
LoggingClass = Any
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class NLPCloudConfig(BaseConfig):
|
| 25 |
+
"""
|
| 26 |
+
Reference: https://docs.nlpcloud.com/#generation
|
| 27 |
+
|
| 28 |
+
- `max_length` (int): Optional. The maximum number of tokens that the generated text should contain.
|
| 29 |
+
|
| 30 |
+
- `length_no_input` (boolean): Optional. Whether `min_length` and `max_length` should not include the length of the input text.
|
| 31 |
+
|
| 32 |
+
- `end_sequence` (string): Optional. A specific token that should be the end of the generated sequence.
|
| 33 |
+
|
| 34 |
+
- `remove_end_sequence` (boolean): Optional. Whether to remove the `end_sequence` string from the result.
|
| 35 |
+
|
| 36 |
+
- `remove_input` (boolean): Optional. Whether to remove the input text from the result.
|
| 37 |
+
|
| 38 |
+
- `bad_words` (list of strings): Optional. List of tokens that are not allowed to be generated.
|
| 39 |
+
|
| 40 |
+
- `temperature` (float): Optional. Temperature sampling. It modulates the next token probabilities.
|
| 41 |
+
|
| 42 |
+
- `top_p` (float): Optional. Top P sampling. Below 1, only the most probable tokens with probabilities that add up to top_p or higher are kept for generation.
|
| 43 |
+
|
| 44 |
+
- `top_k` (int): Optional. Top K sampling. The number of highest probability vocabulary tokens to keep for top k filtering.
|
| 45 |
+
|
| 46 |
+
- `repetition_penalty` (float): Optional. Prevents the same word from being repeated too many times.
|
| 47 |
+
|
| 48 |
+
- `num_beams` (int): Optional. Number of beams for beam search.
|
| 49 |
+
|
| 50 |
+
- `num_return_sequences` (int): Optional. The number of independently computed returned sequences.
|
| 51 |
+
"""
|
| 52 |
+
|
| 53 |
+
max_length: Optional[int] = None
|
| 54 |
+
length_no_input: Optional[bool] = None
|
| 55 |
+
end_sequence: Optional[str] = None
|
| 56 |
+
remove_end_sequence: Optional[bool] = None
|
| 57 |
+
remove_input: Optional[bool] = None
|
| 58 |
+
bad_words: Optional[list] = None
|
| 59 |
+
temperature: Optional[float] = None
|
| 60 |
+
top_p: Optional[float] = None
|
| 61 |
+
top_k: Optional[int] = None
|
| 62 |
+
repetition_penalty: Optional[float] = None
|
| 63 |
+
num_beams: Optional[int] = None
|
| 64 |
+
num_return_sequences: Optional[int] = None
|
| 65 |
+
|
| 66 |
+
def __init__(
|
| 67 |
+
self,
|
| 68 |
+
max_length: Optional[int] = None,
|
| 69 |
+
length_no_input: Optional[bool] = None,
|
| 70 |
+
end_sequence: Optional[str] = None,
|
| 71 |
+
remove_end_sequence: Optional[bool] = None,
|
| 72 |
+
remove_input: Optional[bool] = None,
|
| 73 |
+
bad_words: Optional[list] = None,
|
| 74 |
+
temperature: Optional[float] = None,
|
| 75 |
+
top_p: Optional[float] = None,
|
| 76 |
+
top_k: Optional[int] = None,
|
| 77 |
+
repetition_penalty: Optional[float] = None,
|
| 78 |
+
num_beams: Optional[int] = None,
|
| 79 |
+
num_return_sequences: Optional[int] = None,
|
| 80 |
+
) -> None:
|
| 81 |
+
locals_ = locals().copy()
|
| 82 |
+
for key, value in locals_.items():
|
| 83 |
+
if key != "self" and value is not None:
|
| 84 |
+
setattr(self.__class__, key, value)
|
| 85 |
+
|
| 86 |
+
@classmethod
|
| 87 |
+
def get_config(cls):
|
| 88 |
+
return super().get_config()
|
| 89 |
+
|
| 90 |
+
def validate_environment(
|
| 91 |
+
self,
|
| 92 |
+
headers: dict,
|
| 93 |
+
model: str,
|
| 94 |
+
messages: List[AllMessageValues],
|
| 95 |
+
optional_params: dict,
|
| 96 |
+
litellm_params: dict,
|
| 97 |
+
api_key: Optional[str] = None,
|
| 98 |
+
api_base: Optional[str] = None,
|
| 99 |
+
) -> dict:
|
| 100 |
+
headers = {
|
| 101 |
+
"accept": "application/json",
|
| 102 |
+
"content-type": "application/json",
|
| 103 |
+
}
|
| 104 |
+
if api_key:
|
| 105 |
+
headers["Authorization"] = f"Token {api_key}"
|
| 106 |
+
return headers
|
| 107 |
+
|
| 108 |
+
def get_supported_openai_params(self, model: str) -> List:
|
| 109 |
+
return [
|
| 110 |
+
"max_tokens",
|
| 111 |
+
"stream",
|
| 112 |
+
"temperature",
|
| 113 |
+
"top_p",
|
| 114 |
+
"presence_penalty",
|
| 115 |
+
"frequency_penalty",
|
| 116 |
+
"n",
|
| 117 |
+
"stop",
|
| 118 |
+
]
|
| 119 |
+
|
| 120 |
+
def map_openai_params(
|
| 121 |
+
self,
|
| 122 |
+
non_default_params: dict,
|
| 123 |
+
optional_params: dict,
|
| 124 |
+
model: str,
|
| 125 |
+
drop_params: bool,
|
| 126 |
+
) -> dict:
|
| 127 |
+
for param, value in non_default_params.items():
|
| 128 |
+
if param == "max_tokens":
|
| 129 |
+
optional_params["max_length"] = value
|
| 130 |
+
if param == "stream":
|
| 131 |
+
optional_params["stream"] = value
|
| 132 |
+
if param == "temperature":
|
| 133 |
+
optional_params["temperature"] = value
|
| 134 |
+
if param == "top_p":
|
| 135 |
+
optional_params["top_p"] = value
|
| 136 |
+
if param == "presence_penalty":
|
| 137 |
+
optional_params["presence_penalty"] = value
|
| 138 |
+
if param == "frequency_penalty":
|
| 139 |
+
optional_params["frequency_penalty"] = value
|
| 140 |
+
if param == "n":
|
| 141 |
+
optional_params["num_return_sequences"] = value
|
| 142 |
+
if param == "stop":
|
| 143 |
+
optional_params["stop_sequences"] = value
|
| 144 |
+
return optional_params
|
| 145 |
+
|
| 146 |
+
def get_error_class(
|
| 147 |
+
self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
|
| 148 |
+
) -> BaseLLMException:
|
| 149 |
+
return NLPCloudError(
|
| 150 |
+
status_code=status_code, message=error_message, headers=headers
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
def transform_request(
|
| 154 |
+
self,
|
| 155 |
+
model: str,
|
| 156 |
+
messages: List[AllMessageValues],
|
| 157 |
+
optional_params: dict,
|
| 158 |
+
litellm_params: dict,
|
| 159 |
+
headers: dict,
|
| 160 |
+
) -> dict:
|
| 161 |
+
text = " ".join(convert_content_list_to_str(message) for message in messages)
|
| 162 |
+
|
| 163 |
+
data = {
|
| 164 |
+
"text": text,
|
| 165 |
+
**optional_params,
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
return data
|
| 169 |
+
|
| 170 |
+
def transform_response(
|
| 171 |
+
self,
|
| 172 |
+
model: str,
|
| 173 |
+
raw_response: httpx.Response,
|
| 174 |
+
model_response: ModelResponse,
|
| 175 |
+
logging_obj: LoggingClass,
|
| 176 |
+
request_data: dict,
|
| 177 |
+
messages: List[AllMessageValues],
|
| 178 |
+
optional_params: dict,
|
| 179 |
+
litellm_params: dict,
|
| 180 |
+
encoding: Any,
|
| 181 |
+
api_key: Optional[str] = None,
|
| 182 |
+
json_mode: Optional[bool] = None,
|
| 183 |
+
) -> ModelResponse:
|
| 184 |
+
## LOGGING
|
| 185 |
+
logging_obj.post_call(
|
| 186 |
+
input=None,
|
| 187 |
+
api_key=api_key,
|
| 188 |
+
original_response=raw_response.text,
|
| 189 |
+
additional_args={"complete_input_dict": request_data},
|
| 190 |
+
)
|
| 191 |
+
|
| 192 |
+
## RESPONSE OBJECT
|
| 193 |
+
try:
|
| 194 |
+
completion_response = raw_response.json()
|
| 195 |
+
except Exception:
|
| 196 |
+
raise NLPCloudError(
|
| 197 |
+
message=raw_response.text, status_code=raw_response.status_code
|
| 198 |
+
)
|
| 199 |
+
if "error" in completion_response:
|
| 200 |
+
raise NLPCloudError(
|
| 201 |
+
message=completion_response["error"],
|
| 202 |
+
status_code=raw_response.status_code,
|
| 203 |
+
)
|
| 204 |
+
else:
|
| 205 |
+
try:
|
| 206 |
+
if len(completion_response["generated_text"]) > 0:
|
| 207 |
+
model_response.choices[0].message.content = ( # type: ignore
|
| 208 |
+
completion_response["generated_text"]
|
| 209 |
+
)
|
| 210 |
+
except Exception:
|
| 211 |
+
raise NLPCloudError(
|
| 212 |
+
message=json.dumps(completion_response),
|
| 213 |
+
status_code=raw_response.status_code,
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
## CALCULATING USAGE - baseten charges on time, not tokens - have some mapping of cost here.
|
| 217 |
+
prompt_tokens = completion_response["nb_input_tokens"]
|
| 218 |
+
completion_tokens = completion_response["nb_generated_tokens"]
|
| 219 |
+
|
| 220 |
+
model_response.created = int(time.time())
|
| 221 |
+
model_response.model = model
|
| 222 |
+
usage = Usage(
|
| 223 |
+
prompt_tokens=prompt_tokens,
|
| 224 |
+
completion_tokens=completion_tokens,
|
| 225 |
+
total_tokens=prompt_tokens + completion_tokens,
|
| 226 |
+
)
|
| 227 |
+
setattr(model_response, "usage", usage)
|
| 228 |
+
return model_response
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nlp_cloud/common_utils.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional, Union
|
| 2 |
+
|
| 3 |
+
import httpx
|
| 4 |
+
|
| 5 |
+
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class NLPCloudError(BaseLLMException):
|
| 9 |
+
def __init__(
|
| 10 |
+
self,
|
| 11 |
+
status_code: int,
|
| 12 |
+
message: str,
|
| 13 |
+
headers: Optional[Union[dict, httpx.Headers]] = None,
|
| 14 |
+
):
|
| 15 |
+
super().__init__(status_code=status_code, message=message, headers=headers)
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/novita/chat/transformation.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Support for OpenAI's `/v1/chat/completions` endpoint.
|
| 3 |
+
|
| 4 |
+
Calls done in OpenAI/openai.py as Novita AI is openai-compatible.
|
| 5 |
+
|
| 6 |
+
Docs: https://novita.ai/docs/guides/llm-api
|
| 7 |
+
"""
|
| 8 |
+
|
| 9 |
+
from typing import List, Optional
|
| 10 |
+
|
| 11 |
+
from ....types.llms.openai import AllMessageValues
|
| 12 |
+
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class NovitaConfig(OpenAIGPTConfig):
|
| 16 |
+
def validate_environment(
|
| 17 |
+
self,
|
| 18 |
+
headers: dict,
|
| 19 |
+
model: str,
|
| 20 |
+
messages: List[AllMessageValues],
|
| 21 |
+
optional_params: dict,
|
| 22 |
+
litellm_params: dict,
|
| 23 |
+
api_key: Optional[str] = None,
|
| 24 |
+
api_base: Optional[str] = None,
|
| 25 |
+
) -> dict:
|
| 26 |
+
if api_key is None:
|
| 27 |
+
raise ValueError(
|
| 28 |
+
"Missing Novita AI API Key - A call is being made to novita but no key is set either in the environment variables or via params"
|
| 29 |
+
)
|
| 30 |
+
headers["Authorization"] = f"Bearer {api_key}"
|
| 31 |
+
headers["Content-Type"] = "application/json"
|
| 32 |
+
headers["X-Novita-Source"] = "litellm"
|
| 33 |
+
return headers
|
loan-advisor-agent/.venv/Lib/site-packages/litellm/llms/nscale/chat/transformation.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Optional, Tuple
|
| 2 |
+
|
| 3 |
+
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
|
| 4 |
+
from litellm.secret_managers.main import get_secret_str
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
class NscaleConfig(OpenAIGPTConfig):
|
| 8 |
+
"""
|
| 9 |
+
Reference: Nscale is OpenAI compatible.
|
| 10 |
+
API Key: NSCALE_API_KEY
|
| 11 |
+
Default API Base: https://inference.api.nscale.com/v1
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
API_BASE_URL = "https://inference.api.nscale.com/v1"
|
| 15 |
+
|
| 16 |
+
@property
|
| 17 |
+
def custom_llm_provider(self) -> Optional[str]:
|
| 18 |
+
return "nscale"
|
| 19 |
+
|
| 20 |
+
@staticmethod
|
| 21 |
+
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
|
| 22 |
+
return api_key or get_secret_str("NSCALE_API_KEY")
|
| 23 |
+
|
| 24 |
+
@staticmethod
|
| 25 |
+
def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
|
| 26 |
+
return (
|
| 27 |
+
api_base or get_secret_str("NSCALE_API_BASE") or NscaleConfig.API_BASE_URL
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
def _get_openai_compatible_provider_info(
|
| 31 |
+
self, api_base: Optional[str], api_key: Optional[str]
|
| 32 |
+
) -> Tuple[Optional[str], Optional[str]]:
|
| 33 |
+
# This method is called by get_llm_provider to resolve api_base and api_key
|
| 34 |
+
resolved_api_base = NscaleConfig.get_api_base(api_base)
|
| 35 |
+
resolved_api_key = NscaleConfig.get_api_key(api_key)
|
| 36 |
+
return resolved_api_base, resolved_api_key
|
| 37 |
+
|
| 38 |
+
def get_supported_openai_params(self, model: str) -> list:
|
| 39 |
+
return [
|
| 40 |
+
"max_tokens",
|
| 41 |
+
"n",
|
| 42 |
+
"temperature",
|
| 43 |
+
"top_p",
|
| 44 |
+
"stream",
|
| 45 |
+
"logprobs",
|
| 46 |
+
"top_logprobs",
|
| 47 |
+
"frequency_penalty",
|
| 48 |
+
"presence_penalty",
|
| 49 |
+
"response_format",
|
| 50 |
+
"stop",
|
| 51 |
+
"logit_bias",
|
| 52 |
+
]
|